Makefile: target to for cross platform

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>

.gitignore

@@ -1,4 +1,5 @@
 /.artifacts/
+/.gopathok
 /_output/
 /conmon/conmon.o
 /docs/*.[158]
@@ -16,3 +17,17 @@ Vagrantfile
 .vagrant/
 
 .vscode/
+
+# Eclipse files
+.classpath
+.project
+.settings/**
+
+# Files generated by JetBrains IDEs, e.g. IntelliJ IDEA
+.idea/
+*.iml
+
+# Emacs save files
+*~
+\#*\#
+.\#*

.tool/lint

@@ -5,10 +5,10 @@ set -o nounset
 set -o pipefail
 
 # Create the linter path for use later
-LINTER=${GOPATH}/bin/gometalinter
+LINTER="${LINTER:-${GOPATH}/bin/gometalinter}"
 
 # Make sure gometalinter is installed
-if [ ! -f ${LINTER} ]; then
+if ! command -v ${LINTER} >/dev/null 2>/dev/null; then
 	echo >&2 "gometalinter must be installed. Please run 'make install.tools' and try again"
 	exit 1
 fi

.travis.yml

@@ -32,29 +32,24 @@ jobs:
         - make .gitvalidation
         - make gofmt
         - make lint
-        - make testunit
-        - make docs
-        - make
-      go: 1.8.x
+      go: tip
     - stage: Build and Verify
       script:
-        - make .gitvalidation
-        - make gofmt
-        - make lint
-        - make testunit
-        - make docs
+        - sudo "PATH=$PATH" make testunit
+        - make
+      go: 1.8.x
+    - script:
+        - sudo "PATH=$PATH" make testunit
         - make
       go: 1.9.x
     - script:
-        - make .gitvalidation
-        - make testunit
-        - make docs
+        - sudo "PATH=$PATH" make testunit
         - make
       go: tip
     - stage: Integration Test
       script:
         - make integration
-      go: 1.8.x
+      go: 1.9.x
 
 notifications:
   irc: "chat.freenode.net#cri-o"

Dockerfile

@@ -57,7 +57,7 @@ RUN mkdir -p /usr/src/criu \
     && rm -rf /usr/src/criu
 
 # Install runc
-ENV RUNC_COMMIT c6e4a1ebeb1a72b529c6f1b6ee2b1ae5b868b14f
+ENV RUNC_COMMIT ce80fa0a64803d52883955cb77b2708b438a0b28
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
 	&& git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \
@@ -98,7 +98,7 @@ RUN set -x \
        && rm -rf "$GOPATH"
 
 # Install crictl
-ENV CRICTL_COMMIT b42fc3f364dd48f649d55926c34492beeb9b2e99
+ENV CRICTL_COMMIT 240a840375cdabb5860c75c99e8b0d0a776006b4
 RUN set -x \
        && export GOPATH="$(mktemp -d)" \
        && git clone https://github.com/kubernetes-incubator/cri-tools.git "$GOPATH/src/github.com/kubernetes-incubator/cri-tools" \

Makefile

@@ -1,3 +1,5 @@
+include Makefile.inc
+
 GO ?= go
 EPOCH_TEST_COMMIT ?= 1cc5a27
 PROJECT := github.com/kubernetes-incubator/cri-o
@@ -11,17 +13,16 @@ LIBEXECDIR ?= ${PREFIX}/libexec
 MANDIR ?= ${PREFIX}/share/man
 ETCDIR ?= ${DESTDIR}/etc
 ETCDIR_CRIO ?= ${ETCDIR}/crio
+DATAROOTDIR ?= ${PREFIX}/share/containers
 BUILDTAGS ?= seccomp $(shell hack/btrfs_tag.sh) $(shell hack/libdm_installed.sh) $(shell hack/libdm_no_deferred_remove_tag.sh) $(shell hack/btrfs_installed_tag.sh) $(shell hack/ostree_tag.sh) $(shell hack/selinux_tag.sh)
 CRICTL_CONFIG_DIR=${DESTDIR}/etc
 
 BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions
 OCIUMOUNTINSTALLDIR=$(PREFIX)/share/oci-umount/oci-umount.d
 
-SELINUXOPT ?= $(shell test -x /usr/sbin/selinuxenabled && selinuxenabled && echo -Z)
+SELINUXOPT ?= $(shell selinuxenabled 2>/dev/null && echo -Z)
 PACKAGES ?= $(shell go list -tags "${BUILDTAGS}" ./... | grep -v github.com/kubernetes-incubator/cri-o/vendor)
 
-COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true)
-GIT_COMMIT := $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}")
 BUILD_INFO := $(shell date +%s)
 
 # If GOPATH not specified, use one in the local directory
@@ -47,6 +48,7 @@ help:
 	@echo
 	@echo " * 'install' - Install binaries to system locations"
 	@echo " * 'binaries' - Build crio, conmon and pause"
+	@echo " * 'release-note' - Generate release note"
 	@echo " * 'integration' - Execute integration tests"
 	@echo " * 'clean' - Clean artifacts"
 	@echo " * 'lint' - Execute the source code linter"
@@ -55,7 +57,7 @@ help:
 .gopathok:
 ifeq ("$(wildcard $(GOPKGDIR))","")
 	mkdir -p "$(GOPKGBASEDIR)"
-	ln -s "$(CURDIR)" "$(GOPKGBASEDIR)"
+	ln -s "$(CURDIR)" "$(GOPKGDIR)"
 endif
 	touch "$(GOPATH)/.gopathok"
 
@@ -88,11 +90,15 @@ crio: .gopathok $(shell hack/find-godeps.sh $(GOPKGDIR) cmd/crio $(PROJECT))
 crio.conf: crio
 	./bin/crio --config="" config --default > crio.conf
 
+release-note:
+	@$(GOPATH)/bin/containerd-release -n $(release)
+
 clean:
 ifneq ($(GOPATH),)
 	rm -f "$(GOPATH)/.gopathok"
 endif
 	rm -rf _output
+	rm -rf ./crio.cross.*
 	rm -f docs/*.5 docs/*.8
 	rm -fr test/testdata/redis-image
 	find . -name \*~ -delete
@@ -104,6 +110,17 @@ endif
 	rm -f test/copyimg/copyimg
 	rm -f test/checkseccomp/checkseccomp
 
+
+local-cross:
+	@for target in windows/amd64 darwin/amd64 ; do \
+		os=`echo $${target} | cut -f1 -d/` ; \
+		arch=`echo $${target} | cut -f2 -d/` ; \
+		suffix=$${os}.$${arch} ; \
+		echo "building for $${target} ..."; \
+		GOOS=$${os} GOARCH=$${arch} $(GO) build -i $(LDFLAGS) -tags "containers_image_openpgp" -o crio.cross.$${suffix} $(PROJECT)/cmd/crio && \
+		echo "built ./crio.$${suffix}"; \
+	done
+
 crioimage:
 	docker build -t ${CRIO_IMAGE} .
 
@@ -114,7 +131,7 @@ integration: crioimage
 	docker run -e STORAGE_OPTIONS="--storage-driver=vfs" -e TESTFLAGS -e TRAVIS -t --privileged --rm -v ${CURDIR}:/go/src/${PROJECT} ${CRIO_IMAGE} make localintegration
 
 testunit:
-	$(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES)
+	$(GO) test -tags "$(BUILDTAGS) containers_image_ostree_stub" -cover $(PACKAGES)
 
 localintegration: clean binaries test-binaries
 	./test/test_runner.sh ${TESTFLAGS}
@@ -135,18 +152,19 @@ docs: $(MANPAGES)
 
 install: .gopathok install.bin install.man
 
-install.bin:
+install.bin: binaries
 	install ${SELINUXOPT} -D -m 755 bin/crio $(BINDIR)/crio
 	install ${SELINUXOPT} -D -m 755 bin/conmon $(LIBEXECDIR)/crio/conmon
 	install ${SELINUXOPT} -D -m 755 bin/pause $(LIBEXECDIR)/crio/pause
 
-install.man:
+install.man: $(MANPAGES)
 	install ${SELINUXOPT} -d -m 755 $(MANDIR)/man5
 	install ${SELINUXOPT} -d -m 755 $(MANDIR)/man8
 	install ${SELINUXOPT} -m 644 $(filter %.5,$(MANPAGES)) -t $(MANDIR)/man5
 	install ${SELINUXOPT} -m 644 $(filter %.8,$(MANPAGES)) -t $(MANDIR)/man8
 
 install.config: crio.conf
+	install ${SELINUXOPT} -d $(DATAROOTDIR)/oci/hooks.d
 	install ${SELINUXOPT} -D -m 644 crio.conf $(ETCDIR_CRIO)/crio.conf
 	install ${SELINUXOPT} -D -m 644 seccomp.json $(ETCDIR_CRIO)/seccomp.json
 	install ${SELINUXOPT} -D -m 644 crio-umount.conf $(OCIUMOUNTINSTALLDIR)/crio-umount.conf
@@ -164,9 +182,6 @@ uninstall:
 	rm -f $(BINDIR)/crio
 	rm -f $(LIBEXECDIR)/crio/conmon
 	rm -f $(LIBEXECDIR)/crio/pause
-	for i in $(filter %.1,$(MANPAGES)); do \
-		rm -f $(MANDIR)/man8/$$(basename $${i}); \
-	done
 	for i in $(filter %.5,$(MANPAGES)); do \
 		rm -f $(MANDIR)/man5/$$(basename $${i}); \
 	done
@@ -185,7 +200,12 @@ endif
 
 .PHONY: install.tools
 
-install.tools: .install.gitvalidation .install.gometalinter .install.md2man
+install.tools: .install.gitvalidation .install.gometalinter .install.md2man .install.release
+
+.install.release:
+	if [ ! -x "$(GOPATH)/bin/containerd-release" ]; then \
+		go get -u github.com/containerd/containerd/cmd/containerd-release; \
+	fi
 
 .install.gitvalidation: .gopathok
 	if [ ! -x "$(GOPATH)/bin/git-validation" ]; then \
@@ -216,6 +236,7 @@ install.tools: .install.gitvalidation .install.gometalinter .install.md2man
 
 .PHONY: \
 	binaries \
+	local-cross \
 	clean \
 	conmon \
 	default \

Makefile.inc

@@ -0,0 +1,2 @@
+COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true)
+GIT_COMMIT := $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}")

README.md

@@ -158,28 +158,9 @@ apt-get install -y \
 
 ### Get Source Code
 
-As with other Go projects, CRI-O must be cloned into a directory structure like:
-
-```
-GOPATH
-└── src
-    └── github.com
-        └── kubernetes-incubator
-            └── cri-o
-```
-
-First, configure a `GOPATH` (if you are using go1.8 or later, this defaults to `~/go`).
+Clone the source code using:
 
 ```bash
-export GOPATH=~/go
-mkdir -p $GOPATH
-```
-
-Next, clone the source code using:
-
-```bash
-mkdir -p $GOPATH/src/github.com/kubernetes-incubator
-cd $_ # or cd $GOPATH/src/github.com/kubernetes-incubator
 git clone https://github.com/kubernetes-incubator/cri-o # or your fork
 cd cri-o
 ```
@@ -187,7 +168,6 @@ cd cri-o
 ### Build
 
 ```bash
-make install.tools
 make
 sudo make install
 ```

Vagrantfile

@@ -0,0 +1,110 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://atlas.hashicorp.com/search.
+  config.vm.box = "fedora/27-cloud-base"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  config.vm.synced_folder "./", "/home/vagrant/go/src/github.com/kubernetes-incubator/cri-o", type: "rsync"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+  #   # Display the VirtualBox GUI when booting the machine
+  #   vb.gui = true
+  #
+  #   # Customize the amount of memory on the VM:
+    vb.memory = "4096"
+  end
+  config.vm.provider :libvirt do |libvirt|
+  #   # Customize the amount of memory on the VM:
+    libvirt.memory = "4096"
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
+  # such as FTP and Heroku are also available. See the documentation at
+  # https://docs.vagrantup.com/v2/push/atlas.html for more information.
+  # config.push.define "atlas" do |push|
+  #   push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
+  # end
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  # install pkgs
+  config.vm.provision "shell", inline: <<-SHELL
+    dnf install -y \
+      btrfs-progs-devel \
+      device-mapper-devel \
+      docker \
+      git \
+      glib2-devel \
+      glibc-devel \
+      glibc-static \
+      go \
+      golang-github-cpuguy83-go-md2man \
+      gpgme-devel \
+      libassuan-devel \
+      libgpg-error-devel \
+      libseccomp-devel \
+      libselinux-devel \
+      ostree-devel \
+      pkgconfig \
+      runc \
+      skopeo-containers
+    chown vagrant:vagrant -R /home/vagrant
+    modprobe overlay
+  SHELL
+  config.vm.provision "shell", privileged: true, inline: <<-SHELL
+    groupadd docker
+    usermod -aG docker vagrant
+    systemctl start docker.service
+  SHELL
+  config.vm.provision "shell", privileged: false, inline: <<-SHELL
+    export GOPATH=/home/vagrant/go
+    SRCDIR=$GOPATH/src/github.com/kubernetes-incubator/cri-o
+    echo "export GOPATH=$GOPATH" >> $HOME/.bashrc
+    echo "export PATH=$PATH:$GOPATH/bin" >> $HOME/.bashrc
+    # convenience: drop into the cri-o dir when login to vm
+    echo "cd $SRCDIR" >> $HOME/.bashrc
+    cd $SRCDIR
+    make
+  SHELL
+end

cmd/crio/config.go

@@ -114,9 +114,6 @@ default_mounts = [
 # pids_limit is the number of processes allowed in a container
 pids_limit = {{ .PidsLimit }}
 
-# enable using a shared PID namespace for containers in a pod
-enable_shared_pid_namespace = {{ .EnableSharedPIDNamespace }}
-
 # log_size_max is the max limit for the container log size in bytes.
 # Negative values indicate that no limit is imposed.
 log_size_max = {{ .LogSizeMax }}

cmd/crio/daemon_unsupported.go

@@ -0,0 +1,7 @@
+// +build !linux
+
+package main
+
+func notifySystem() {
+	// nothin' doin'
+}

cmd/crio/main.go

@@ -11,19 +11,18 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/containers/storage/pkg/reexec"
 	"github.com/kubernetes-incubator/cri-o/lib"
 	"github.com/kubernetes-incubator/cri-o/server"
 	"github.com/kubernetes-incubator/cri-o/version"
-	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/sirupsen/logrus"
 	"github.com/soheilhy/cmux"
 	"github.com/urfave/cli"
-	"golang.org/x/sys/unix"
 	"google.golang.org/grpc"
-	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // gitCommit is the commit that the binary is being built from.
@@ -132,9 +131,6 @@ func mergeConfig(config *server.Config, ctx *cli.Context) error {
 	if ctx.GlobalIsSet("pids-limit") {
 		config.PidsLimit = ctx.GlobalInt64("pids-limit")
 	}
-	if ctx.GlobalIsSet("enable-shared-pid-namespace") {
-		config.EnableSharedPIDNamespace = ctx.GlobalBool("enable-shared-pid-namespace")
-	}
 	if ctx.GlobalIsSet("log-size-max") {
 		config.LogSizeMax = ctx.GlobalInt64("log-size-max")
 	}
@@ -152,13 +148,13 @@ func mergeConfig(config *server.Config, ctx *cli.Context) error {
 
 func catchShutdown(gserver *grpc.Server, sserver *server.Server, hserver *http.Server, signalled *bool) {
 	sig := make(chan os.Signal, 10)
-	signal.Notify(sig, unix.SIGINT, unix.SIGTERM)
+	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
 	go func() {
 		for s := range sig {
 			switch s {
-			case unix.SIGINT:
+			case syscall.SIGINT:
 				logrus.Debugf("Caught SIGINT")
-			case unix.SIGTERM:
+			case syscall.SIGTERM:
 				logrus.Debugf("Caught SIGTERM")
 			default:
 				continue
@@ -167,7 +163,7 @@ func catchShutdown(gserver *grpc.Server, sserver *server.Server, hserver *http.S
 			gserver.GracefulStop()
 			hserver.Shutdown(context.Background())
 			sserver.StopStreamServer()
-			sserver.StopExitMonitor()
+			sserver.StopMonitors()
 			if err := sserver.Shutdown(); err != nil {
 				logrus.Warnf("error shutting down main service %v", err)
 			}
@@ -300,10 +296,6 @@ func main() {
 			Value: lib.DefaultPidsLimit,
 			Usage: "maximum number of processes allowed in a container",
 		},
-		cli.BoolFlag{
-			Name:  "enable-shared-pid-namespace",
-			Usage: "enable using a shared PID namespace for containers in a pod",
-		},
 		cli.Int64Flag{
 			Name:  "log-size-max",
 			Value: lib.DefaultLogSizeMax,
@@ -429,7 +421,7 @@ func main() {
 		config := c.App.Metadata["config"].(*server.Config)
 
 		if !config.SELinux {
-			selinux.SetDisabled()
+			disableSELinux()
 		}
 
 		if _, err := os.Stat(config.Runtime); os.IsNotExist(err) {
@@ -485,6 +477,9 @@ func main() {
 		go func() {
 			service.StartExitMonitor()
 		}()
+		go func() {
+			service.StartHooksMonitor()
+		}()
 
 		m := cmux.New(lis)
 		grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
@@ -515,10 +510,10 @@ func main() {
 		}()
 
 		streamServerCloseCh := service.StreamingServerCloseChan()
-		serverExitMonitorCh := service.ExitMonitorCloseChan()
+		serverMonitorsCh := service.MonitorsCloseChan()
 		select {
 		case <-streamServerCloseCh:
-		case <-serverExitMonitorCh:
+		case <-serverMonitorsCh:
 		case <-serverCloseCh:
 		}
 
@@ -526,8 +521,8 @@ func main() {
 
 		<-streamServerCloseCh
 		logrus.Debug("closed stream server")
-		<-serverExitMonitorCh
-		logrus.Debug("closed exit monitor")
+		<-serverMonitorsCh
+		logrus.Debug("closed monitors")
 		<-serverCloseCh
 		logrus.Debug("closed main server")
 

cmd/crio/selinux_linux.go

@@ -0,0 +1,9 @@
+// +build linux
+
+package main
+
+import selinux "github.com/opencontainers/selinux/go-selinux"
+
+func disableSELinux() {
+	selinux.SetDisabled()
+}

cmd/crio/selinux_unsupported.go

@@ -0,0 +1,9 @@
+// +build !linux
+
+package main
+
+import "github.com/sirupsen/logrus"
+
+func disableSELinux() {
+	logrus.Infof("there is no selinux to disable")
+}

conmon/Makefile

@@ -1,8 +1,13 @@
+include ../Makefile.inc
+
 src = $(wildcard *.c)
 obj = $(src:.c=.o)
 
 override LIBS += $(shell pkg-config --libs glib-2.0)
-override CFLAGS += -std=c99 -Os -Wall -Wextra $(shell pkg-config --cflags glib-2.0)
+
+VERSION = $(shell sed -n -e 's/^const Version = "\([^"]*\)"/\1/p' ../version/version.go)
+
+override CFLAGS += -std=c99 -Os -Wall -Wextra $(shell pkg-config --cflags glib-2.0) -DVERSION=\"$(VERSION)\" -DGIT_COMMIT=\"$(GIT_COMMIT)\"
 
 conmon: $(obj)
 	$(CC) -o ../bin/$@ $^ $(CFLAGS) $(LIBS)

conmon/cmsg.c

@@ -26,12 +26,19 @@
 
 #include "cmsg.h"
 
-#define error(fmt, ...)							\
-	({								\
-		fprintf(stderr, "nsenter: " fmt ": %m\n", ##__VA_ARGS__); \
-		errno = ECOMM;						\
-		goto err; /* return value */				\
-	})
+#define error(s)								\
+	do {									\
+		fprintf(stderr, "nsenter: %s %s\n", s, strerror(errno));	\
+		errno = ECOMM;							\
+		goto err; /* return value */					\
+	} while (0)
+
+#define errorf(fmt, ...)									\
+	do {											\
+		fprintf(stderr, "nsenter: " fmt ": %s\n", ##__VA_ARGS__, strerror(errno));	\
+		errno = ECOMM;									\
+		goto err; /* return value */							\
+	} while (0)
 
 /*
  * Sends a file descriptor along the sockfd provided. Returns the return
@@ -103,7 +110,7 @@ struct file_t recvfd(int sockfd)
 	/* TODO: Make this dynamic with MSG_PEEK. */
 	file.name = malloc(TAG_BUFFER);
 	if (!file.name)
-		error("recvfd: failed to allocate file.tag buffer\n");
+		error("recvfd: failed to allocate file.tag buffer");
 
 	/*
 	 * We need to "recieve" the non-ancillary data even though we don't
@@ -128,11 +135,11 @@ struct file_t recvfd(int sockfd)
 	if (!cmsg)
 		error("recvfd: got NULL from CMSG_FIRSTHDR");
 	if (cmsg->cmsg_level != SOL_SOCKET)
-		error("recvfd: expected SOL_SOCKET in cmsg: %d", cmsg->cmsg_level);
+		errorf("recvfd: expected SOL_SOCKET in cmsg: %d", cmsg->cmsg_level);
 	if (cmsg->cmsg_type != SCM_RIGHTS)
-		error("recvfd: expected SCM_RIGHTS in cmsg: %d", cmsg->cmsg_type);
+		errorf("recvfd: expected SCM_RIGHTS in cmsg: %d", cmsg->cmsg_type);
 	if (cmsg->cmsg_len != CMSG_LEN(sizeof(int)))
-		error("recvfd: expected correct CMSG_LEN in cmsg: %lu", cmsg->cmsg_len);
+		errorf("recvfd: expected correct CMSG_LEN in cmsg: %lu", cmsg->cmsg_len);
 
 	fdptr = (int *) CMSG_DATA(cmsg);
 	if (!fdptr || *fdptr < 0)

conmon/conmon.c

@@ -27,27 +27,53 @@
 
 #include "cmsg.h"
 
-#define pexit(fmt, ...)                                                          \
-	do {                                                                     \
-		fprintf(stderr, "[conmon:e]: " fmt " %m\n", ##__VA_ARGS__);      \
-		syslog(LOG_ERR, "conmon <error>: " fmt ": %m\n", ##__VA_ARGS__); \
-		exit(EXIT_FAILURE);                                              \
+#define pexit(s)                                                                \
+	do {                                                                    \
+		fprintf(stderr, "[conmon:e]: %s %s\n", s, strerror(errno));     \
+		syslog(LOG_ERR, "conmon <error>: %s %s\n", s, strerror(errno)); \
+		exit(EXIT_FAILURE);                                             \
 	} while (0)
 
-#define nexit(fmt, ...)                                                       \
+#define pexitf(fmt, ...)                                                                          \
+	do {                                                                                      \
+		fprintf(stderr, "[conmon:e]: " fmt " %s\n", ##__VA_ARGS__, strerror(errno));      \
+		syslog(LOG_ERR, "conmon <error>: " fmt ": %s\n", ##__VA_ARGS__, strerror(errno)); \
+		exit(EXIT_FAILURE);                                                               \
+	} while (0)
+
+#define nexit(s)                                            \
+	do {                                                \
+		fprintf(stderr, "[conmon:e] %s\n", s);      \
+		syslog(LOG_ERR, "conmon <error>: %s\n", s); \
+		exit(EXIT_FAILURE);                         \
+	} while (0)
+
+#define nexitf(fmt, ...)                                                      \
 	do {                                                                  \
 		fprintf(stderr, "[conmon:e]: " fmt "\n", ##__VA_ARGS__);      \
 		syslog(LOG_ERR, "conmon <error>: " fmt " \n", ##__VA_ARGS__); \
 		exit(EXIT_FAILURE);                                           \
 	} while (0)
 
-#define nwarn(fmt, ...)                                                        \
+#define nwarn(s)                                             \
+	do {                                                 \
+		fprintf(stderr, "[conmon:w]: %s\n", s);      \
+		syslog(LOG_INFO, "conmon <nwarn>: %s\n", s); \
+	} while (0)
+
+#define nwarnf(fmt, ...)                                                       \
 	do {                                                                   \
 		fprintf(stderr, "[conmon:w]: " fmt "\n", ##__VA_ARGS__);       \
 		syslog(LOG_INFO, "conmon <nwarn>: " fmt " \n", ##__VA_ARGS__); \
 	} while (0)
 
-#define ninfo(fmt, ...)                                                        \
+#define ninfo(s)                                             \
+	do {                                                 \
+		fprintf(stderr, "[conmon:i]: %s\n", s);      \
+		syslog(LOG_INFO, "conmon <ninfo>: %s\n", s); \
+	} while (0)
+
+#define ninfof(fmt, ...)                                                       \
 	do {                                                                   \
 		fprintf(stderr, "[conmon:i]: " fmt "\n", ##__VA_ARGS__);       \
 		syslog(LOG_INFO, "conmon <ninfo>: " fmt " \n", ##__VA_ARGS__); \
@@ -97,8 +123,12 @@ static inline void strv_cleanup(char ***strv)
 
 #define DEFAULT_SOCKET_PATH "/var/lib/crio"
 
+static volatile pid_t container_pid = -1;
+static volatile pid_t create_pid = -1;
+static bool opt_version = false;
 static bool opt_terminal = false;
 static bool opt_stdin = false;
+static bool opt_leave_stdin_open = false;
 static char *opt_cid = NULL;
 static char *opt_cuuid = NULL;
 static char *opt_runtime_path = NULL;
@@ -113,24 +143,34 @@ static char *opt_exit_dir = NULL;
 static int opt_timeout = 0;
 static int64_t opt_log_size_max = -1;
 static char *opt_socket_path = DEFAULT_SOCKET_PATH;
+static bool opt_no_new_keyring = false;
+static char *opt_exit_command = NULL;
+static gchar **opt_exit_args = NULL;
+static bool opt_replace_listen_pid = false;
 static GOptionEntry opt_entries[] =
 {
   { "terminal", 't', 0, G_OPTION_ARG_NONE, &opt_terminal, "Terminal", NULL },
   { "stdin", 'i', 0, G_OPTION_ARG_NONE, &opt_stdin, "Stdin", NULL },
+  { "leave-stdin-open", 0, 0, G_OPTION_ARG_NONE, &opt_leave_stdin_open, "Leave stdin open when attached client disconnects", NULL },
   { "cid", 'c', 0, G_OPTION_ARG_STRING, &opt_cid, "Container ID", NULL },
   { "cuuid", 'u', 0, G_OPTION_ARG_STRING, &opt_cuuid, "Container UUID", NULL },
   { "runtime", 'r', 0, G_OPTION_ARG_STRING, &opt_runtime_path, "Runtime path", NULL },
-  { "no-pivot", 0, 0, G_OPTION_ARG_NONE, &opt_no_pivot, "do not use pivot_root", NULL },
+  { "no-new-keyring", 0, 0, G_OPTION_ARG_NONE, &opt_no_new_keyring, "Do not create a new session keyring for the container", NULL },
+  { "no-pivot", 0, 0, G_OPTION_ARG_NONE, &opt_no_pivot, "Do not use pivot_root", NULL },
+  { "replace-listen-pid", 0, 0, G_OPTION_ARG_NONE, &opt_replace_listen_pid, "Replace listen pid if set for oci-runtime pid", NULL },
   { "bundle", 'b', 0, G_OPTION_ARG_STRING, &opt_bundle_path, "Bundle path", NULL },
   { "pidfile", 'p', 0, G_OPTION_ARG_STRING, &opt_pid_file, "PID file", NULL },
   { "systemd-cgroup", 's', 0, G_OPTION_ARG_NONE, &opt_systemd_cgroup, "Enable systemd cgroup manager", NULL },
   { "exec", 'e', 0, G_OPTION_ARG_NONE, &opt_exec, "Exec a command in a running container", NULL },
   { "exec-process-spec", 0, 0, G_OPTION_ARG_STRING, &opt_exec_process_spec, "Path to the process spec for exec", NULL },
   { "exit-dir", 0, 0, G_OPTION_ARG_STRING, &opt_exit_dir, "Path to the directory where exit files are written", NULL },
+  { "exit-command", 0, 0, G_OPTION_ARG_STRING, &opt_exit_command, "Path to the program to execute when the container terminates its execution", NULL },
+  { "exit-command-arg", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_exit_args, "Additional arg to pass to the exit command.  Can be specified multiple times", NULL },
   { "log-path", 'l', 0, G_OPTION_ARG_STRING, &opt_log_path, "Log file path", NULL },
   { "timeout", 'T', 0, G_OPTION_ARG_INT, &opt_timeout, "Timeout in seconds", NULL },
   { "log-size-max", 0, 0, G_OPTION_ARG_INT64, &opt_log_size_max, "Maximum size of log file", NULL },
   { "socket-dir-path", 0, 0, G_OPTION_ARG_STRING, &opt_socket_path, "Location of container attach sockets", NULL },
+  { "version", 0, 0, G_OPTION_ARG_NONE, &opt_version, "Print the version and exit", NULL },
   { NULL }
 };
 
@@ -335,21 +375,23 @@ static int write_k8s_log(int fd, stdpipe_t pipe, const char *buf, ssize_t buflen
 		 * a timestamp.
 		 */
 		if ((opt_log_size_max > 0) && (bytes_written + bytes_to_be_written) > opt_log_size_max) {
+			_cleanup_free_ char *opt_log_path_tmp = g_strdup_printf("%s.tmp", opt_log_path);
 			ninfo("Creating new log file");
 			bytes_written = 0;
 
 			/* Close the existing fd */
 			close(fd);
 
-			/* Unlink the file */
-			if (unlink(opt_log_path) < 0) {
-				pexit("Failed to unlink log file");
+			/* Open the log path file again */
+			log_fd = open(opt_log_path_tmp, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, 0600);
+			if (log_fd < 0)
+				pexitf("Failed to open log file %s", opt_log_path);
+
+			/* Replace the previous file */
+			if (rename(opt_log_path_tmp, opt_log_path) < 0) {
+				pexit("Failed to rename log file");
 			}
 
-			/* Open the log path file again */
-			log_fd = open(opt_log_path, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC, 0600);
-			if (log_fd < 0)
-				pexit("Failed to open log file %s: %s", opt_log_path, strerror(errno));
 			fd = log_fd;
 		}
 
@@ -397,7 +439,7 @@ next:
 		nwarn("failed to flush buffer to log");
 	}
 
-	ninfo("Total bytes written: %"PRId64"", bytes_written);
+	ninfof("Total bytes written: %"PRId64"", bytes_written);
 
 	return 0;
 }
@@ -411,7 +453,7 @@ static char *process_cgroup_subsystem_path(int pid, const char *subsystem) {
 	_cleanup_fclose_ FILE *fp = NULL;
 	fp = fopen(cgroups_file_path, "re");
 	if (fp == NULL) {
-		nwarn("Failed to open cgroups file: %s", cgroups_file_path);
+		nwarnf("Failed to open cgroups file: %s", cgroups_file_path);
 		return NULL;
 	}
 
@@ -425,13 +467,13 @@ static char *process_cgroup_subsystem_path(int pid, const char *subsystem) {
 		_cleanup_strv_ char **subsystems = NULL;
 		ptr = strchr(line, ':');
 		if (ptr == NULL) {
-			nwarn("Error parsing cgroup, ':' not found: %s", line);
+			nwarnf("Error parsing cgroup, ':' not found: %s", line);
 			return NULL;
 		}
 		ptr++;
 		path = strchr(ptr, ':');
 		if (path == NULL) {
-			nwarn("Error parsing cgroup, second ':' not found: %s", line);
+			nwarnf("Error parsing cgroup, second ':' not found: %s", line);
 			return NULL;
 		}
 		*path = 0;
@@ -495,9 +537,9 @@ static int get_pipe_fd_from_env(const char *envname)
 	errno = 0;
 	pipe_fd = strtol(pipe_str, &endptr, 10);
 	if (errno != 0 || *endptr != '\0')
-		pexit("unable to parse %s", envname);
+		pexitf("unable to parse %s", envname);
 	if (fcntl(pipe_fd, F_SETFD, FD_CLOEXEC) == -1)
-		pexit("unable to make %s CLOEXEC", envname);
+		pexitf("unable to make %s CLOEXEC", envname);
 
 	return pipe_fd;
 }
@@ -587,7 +629,7 @@ static bool read_stdio(int fd, stdpipe_t pipe, bool *eof)
 			*eof = true;
 		return false;
 	} else if (num_read < 0) {
-		nwarn("stdio_input read failed %s", strerror(errno));
+		nwarnf("stdio_input read failed %s", strerror(errno));
 		return false;
 	} else {
 		if (write_k8s_log(log_fd, pipe, buf, num_read) < 0) {
@@ -606,7 +648,25 @@ static bool read_stdio(int fd, stdpipe_t pipe, bool *eof)
 
 static void on_sigchld(G_GNUC_UNUSED int signal)
 {
-	raise (SIGUSR1);
+	raise(SIGUSR1);
+}
+
+static void on_sig_exit(int signal)
+{
+	if (container_pid > 0) {
+		if (kill(container_pid, signal) == 0)
+			return;
+	} else if (create_pid > 0) {
+		if (kill(create_pid, signal) == 0)
+			return;
+		if (errno == ESRCH) {
+			/* The create_pid process might have exited, so try container_pid again.  */
+			if (container_pid > 0 && kill(container_pid, signal) == 0)
+				return;
+		}
+	}
+	/* Just force a check if we get here.  */
+	raise(SIGUSR1);
 }
 
 static void check_child_processes(GHashTable *pid_to_handler)
@@ -755,8 +815,12 @@ static gboolean conn_sock_cb(int fd, GIOCondition condition, G_GNUC_UNUSED gpoin
 	/* End of input */
 	conn_sock_shutdown(SHUT_RD);
 	if (masterfd_stdin >= 0 && opt_stdin) {
-		close(masterfd_stdin);
-		masterfd_stdin = -1;
+		if (!opt_leave_stdin_open) {
+			close(masterfd_stdin);
+			masterfd_stdin = -1;
+		} else {
+			ninfo("Not closing input");
+		}
 	}
 	return G_SOURCE_REMOVE;
 }
@@ -771,7 +835,7 @@ static gboolean attach_cb(int fd, G_GNUC_UNUSED GIOCondition condition, G_GNUC_U
 		conn_sock_readable = true;
 		conn_sock_writable = true;
 		g_unix_fd_add (conn_sock, G_IO_IN|G_IO_HUP|G_IO_ERR, conn_sock_cb, GINT_TO_POINTER(STDOUT_PIPE));
-		ninfo("Accepted connection %d", conn_sock);
+		ninfof("Accepted connection %d", conn_sock);
 	}
 
 	return G_SOURCE_CONTINUE;
@@ -797,7 +861,7 @@ static gboolean ctrl_cb(int fd, G_GNUC_UNUSED GIOCondition condition, G_GNUC_UNU
 	}
 
 	readptr[num_read] = '\0';
-	ninfo("Got ctl message: %s\n", ctlbuf);
+	ninfof("Got ctl message: %s", ctlbuf);
 
 	char *beg = ctlbuf;
 	char *newline = strchrnul(beg, '\n');
@@ -808,9 +872,9 @@ static gboolean ctrl_cb(int fd, G_GNUC_UNUSED GIOCondition condition, G_GNUC_UNU
 			nwarn("Failed to sscanf message");
 			return G_SOURCE_CONTINUE;
 		}
-		ninfo("Message type: %d, Height: %d, Width: %d", ctl_msg_type, height, width);
+		ninfof("Message type: %d, Height: %d, Width: %d", ctl_msg_type, height, width);
 		ret = ioctl(masterfd_stdout, TIOCGWINSZ, &ws);
-		ninfo("Existing size: %d %d", ws.ws_row, ws.ws_col);
+		ninfof("Existing size: %d %d", ws.ws_row, ws.ws_col);
 		ws.ws_row = height;
 		ws.ws_col = width;
 		ret = ioctl(masterfd_stdout, TIOCSWINSZ, &ws);
@@ -826,7 +890,7 @@ static gboolean ctrl_cb(int fd, G_GNUC_UNUSED GIOCondition condition, G_GNUC_UNU
 		 * This shouldn't happen as our buffer is larger than
 		 * the message that we expect to receive.
 		 */
-		nwarn("Could not find newline in entire buffer\n");
+		nwarn("Could not find newline in entire buffer");
 	} else if (*beg == '\0') {
 		/* We exhausted all messages that were complete */
 		readptr = ctlbuf;
@@ -854,7 +918,7 @@ static gboolean terminal_accept_cb(int fd, G_GNUC_UNUSED GIOCondition condition,
 	int connfd = -1;
 	struct termios tset;
 
-	ninfo("about to accept from console_socket_fd: %d", fd);
+	ninfof("about to accept from console_socket_fd: %d", fd);
 	connfd = accept4(fd, NULL, NULL, SOCK_CLOEXEC);
 	if (connfd < 0) {
 		nwarn("Failed to accept console-socket connection");
@@ -866,10 +930,10 @@ static gboolean terminal_accept_cb(int fd, G_GNUC_UNUSED GIOCondition condition,
 	unlink(csname);
 
 	/* We exit if this fails. */
-	ninfo("about to recvfd from connfd: %d", connfd);
+	ninfof("about to recvfd from connfd: %d", connfd);
 	console = recvfd(connfd);
 
-	ninfo("console = {.name = '%s'; .fd = %d}", console.name, console.fd);
+	ninfof("console = {.name = '%s'; .fd = %d}", console.name, console.fd);
 	free(console.name);
 
 	/* We change the terminal settings to match kube settings */
@@ -896,14 +960,16 @@ static void
 runtime_exit_cb (G_GNUC_UNUSED GPid pid, int status, G_GNUC_UNUSED gpointer user_data)
 {
 	runtime_status = status;
+	create_pid = -1;
 	g_main_loop_quit (main_loop);
 }
 
 static void
 container_exit_cb (G_GNUC_UNUSED GPid pid, int status, G_GNUC_UNUSED gpointer user_data)
 {
-	ninfo("container %d exited with status %d\n", pid, status);
+	ninfof("container %d exited with status %d", pid, status);
 	container_status = status;
+	container_pid = -1;
 	g_main_loop_quit (main_loop);
 }
 
@@ -953,7 +1019,7 @@ static char *setup_console_socket(void)
 	addr.sun_family = AF_UNIX;
 	strncpy(addr.sun_path, csname, sizeof(addr.sun_path)-1);
 
-	ninfo("addr{sun_family=AF_UNIX, sun_path=%s}", addr.sun_path);
+	ninfof("addr{sun_family=AF_UNIX, sun_path=%s}", addr.sun_path);
 
 	/* Bind to the console socket path. */
 	console_socket_fd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0);
@@ -991,10 +1057,10 @@ static char *setup_attach_socket(void)
 		pexit("Failed to create symlink for attach socket");
 
 	attach_sock_path = g_build_filename(opt_socket_path, opt_cuuid, "attach", NULL);
-	ninfo("attach sock path: %s", attach_sock_path);
+	ninfof("attach sock path: %s", attach_sock_path);
 
 	strncpy(attach_addr.sun_path, attach_sock_path, sizeof(attach_addr.sun_path) - 1);
-	ninfo("addr{sun_family=AF_UNIX, sun_path=%s}", attach_addr.sun_path);
+	ninfof("addr{sun_family=AF_UNIX, sun_path=%s}", attach_addr.sun_path);
 
 	/*
 	 * We make the socket non-blocking to avoid a race where client aborts connection
@@ -1009,10 +1075,10 @@ static char *setup_attach_socket(void)
 		pexit("Failed to change attach socket permissions");
 
 	if (bind(attach_socket_fd, (struct sockaddr *)&attach_addr, sizeof(struct sockaddr_un)) == -1)
-		pexit("Failed to bind attach socket: %s", attach_sock_path);
+		pexitf("Failed to bind attach socket: %s", attach_sock_path);
 
 	if (listen(attach_socket_fd, 10) == -1)
-		pexit("Failed to listen on attach socket: %s", attach_sock_path);
+		pexitf("Failed to listen on attach socket: %s", attach_sock_path);
 
 	g_unix_fd_add (attach_socket_fd, G_IO_IN, attach_cb, NULL);
 
@@ -1022,12 +1088,12 @@ static char *setup_attach_socket(void)
 static void setup_terminal_control_fifo()
 {
 	_cleanup_free_ char *ctl_fifo_path = g_build_filename(opt_bundle_path, "ctl", NULL);
-	ninfo("ctl fifo path: %s", ctl_fifo_path);
+	ninfof("ctl fifo path: %s", ctl_fifo_path);
 
 	/* Setup fifo for reading in terminal resize and other stdio control messages */
 
 	if (mkfifo(ctl_fifo_path, 0666) == -1)
-		pexit("Failed to mkfifo at %s", ctl_fifo_path);
+		pexitf("Failed to mkfifo at %s", ctl_fifo_path);
 
 	terminal_ctrl_fd = open(ctl_fifo_path, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
 	if (terminal_ctrl_fd == -1)
@@ -1043,7 +1109,7 @@ static void setup_terminal_control_fifo()
 
 	g_unix_fd_add (terminal_ctrl_fd, G_IO_IN, ctrl_cb, NULL);
 
-	ninfo("terminal_ctrl_fd: %d", terminal_ctrl_fd);
+	ninfof("terminal_ctrl_fd: %d", terminal_ctrl_fd);
 }
 
 static void setup_oom_handling(int container_pid)
@@ -1059,13 +1125,13 @@ static void setup_oom_handling(int container_pid)
 	_cleanup_free_ char *memory_cgroup_file_path = g_build_filename(memory_cgroup_path, "cgroup.event_control", NULL);
 
 	if ((cfd = open(memory_cgroup_file_path, O_WRONLY | O_CLOEXEC)) == -1) {
-		nwarn("Failed to open %s", memory_cgroup_file_path);
+		nwarnf("Failed to open %s", memory_cgroup_file_path);
 		return;
 	}
 
 	_cleanup_free_ char *memory_cgroup_file_oom_path = g_build_filename(memory_cgroup_path, "memory.oom_control", NULL);
 	if ((ofd = open(memory_cgroup_file_oom_path, O_RDONLY | O_CLOEXEC)) == -1)
-		pexit("Failed to open %s", memory_cgroup_file_oom_path);
+		pexitf("Failed to open %s", memory_cgroup_file_oom_path);
 
 	if ((oom_event_fd = eventfd(0, EFD_CLOEXEC)) == -1)
 		pexit("Failed to create eventfd");
@@ -1077,6 +1143,31 @@ static void setup_oom_handling(int container_pid)
 	g_unix_fd_add (oom_event_fd, G_IO_IN, oom_cb, NULL);
 }
 
+static void do_exit_command()
+{
+	gchar **args;
+	size_t n_args = 0;
+
+	/* Count the additional args, if any.  */
+	if (opt_exit_args)
+		for (; opt_exit_args[n_args]; n_args++);
+
+	args = malloc(sizeof (gchar *) * (n_args + 2));
+	if (args == NULL)
+		_exit(EXIT_FAILURE);
+
+	args[0] = opt_exit_command;
+	if (opt_exit_args)
+		for (n_args = 0; opt_exit_args[n_args]; n_args++)
+			args[n_args + 1] = opt_exit_args[n_args];
+	args[n_args + 1] = NULL;
+
+	execve(opt_exit_command, args, NULL);
+
+        /* Should not happen, but better be safe. */
+	_exit(EXIT_FAILURE);
+}
+
 int main(int argc, char *argv[])
 {
 	int ret;
@@ -1085,8 +1176,7 @@ int main(int argc, char *argv[])
 	_cleanup_free_ char *csname = NULL;
 	GError *err = NULL;
 	_cleanup_free_ char *contents = NULL;
-	int container_pid = -1;
-	pid_t main_pid, create_pid;
+	pid_t main_pid;
 	/* Used for !terminal cases. */
 	int slavefd_stdin = -1;
 	int slavefd_stdout = -1;
@@ -1111,6 +1201,10 @@ int main(int argc, char *argv[])
 	        g_print("option parsing failed: %s\n", error->message);
 	        exit(1);
 	}
+	if (opt_version) {
+		g_print("conmon version " VERSION "\ncommit: " GIT_COMMIT "\n");
+		exit(0);
+	}
 
 	if (opt_cid == NULL)
 		nexit("Container ID not provided. Use --cid");
@@ -1121,10 +1215,7 @@ int main(int argc, char *argv[])
 	if (opt_runtime_path == NULL)
 		nexit("Runtime path not provided. Use --runtime");
 	if (access(opt_runtime_path, X_OK) < 0)
-		pexit("Runtime path %s is not valid: %s", opt_runtime_path, strerror(errno));
-
-	if (!opt_exec && opt_exit_dir == NULL)
-		nexit("Container exit directory not provided. Use --exit-dir");
+		pexitf("Runtime path %s is not valid", opt_runtime_path);
 
 	if (opt_bundle_path == NULL && !opt_exec) {
 		if (getcwd(cwd, sizeof(cwd)) == NULL) {
@@ -1158,7 +1249,10 @@ int main(int argc, char *argv[])
 		/* Block for an initial write to the start pipe before
 		   spawning any childred or exiting, to ensure the
 		   parent can put us in the right cgroup. */
-		read(start_pipe_fd, buf, BUF_SIZE);
+		num_read = read(start_pipe_fd, buf, BUF_SIZE);
+		if (num_read < 0) {
+			pexit("start-pipe read failed");
+		}
 		close(start_pipe_fd);
 	}
 
@@ -1269,6 +1363,12 @@ int main(int argc, char *argv[])
 			NULL);
 	}
 
+	if (!opt_exec && opt_no_new_keyring) {
+		add_argv(runtime_argv,
+			"--no-new-keyring",
+			NULL);
+	}
+
 	if (csname != NULL) {
 		add_argv(runtime_argv,
 			 "--console-socket", csname,
@@ -1286,6 +1386,10 @@ int main(int argc, char *argv[])
 	add_argv(runtime_argv, opt_cid, NULL);
 	end_argv(runtime_argv);
 
+	sigset_t mask, oldmask;
+	if ((sigemptyset(&mask) < 0) || (sigaddset(&mask, SIGTERM) < 0) || (sigaddset(&mask, SIGQUIT) < 0)
+		|| (sigaddset(&mask, SIGINT) < 0) || sigprocmask (SIG_BLOCK, &mask, &oldmask) < 0)
+		pexit("Failed to block signals");
 	/*
 	 * We have to fork here because the current runC API dups the stdio of the
 	 * calling process over the container's fds. This is actually *very bad*
@@ -1300,6 +1404,11 @@ int main(int argc, char *argv[])
 		pexit("Failed to fork the create command");
 	} else if (!create_pid) {
 		/* FIXME: This results in us not outputting runc error messages to crio's log. */
+		if (prctl(PR_SET_PDEATHSIG, SIGKILL) < 0)
+			pexit("Failed to set PDEATHSIG");
+		if (sigprocmask (SIG_SETMASK, &oldmask, NULL) < 0)
+			pexit("Failed to unblock signals");
+
 		if (slavefd_stdin < 0)
 			slavefd_stdin = dev_null_r;
 		if (dup2(slavefd_stdin, STDIN_FILENO) < 0)
@@ -1315,10 +1424,37 @@ int main(int argc, char *argv[])
 		if (dup2(slavefd_stderr, STDERR_FILENO) < 0)
 			pexit("Failed to dup over stderr");
 
+		/* If LISTEN_PID env is set, we need to set the LISTEN_PID
+		   it to the new child process */
+		char *listenpid = getenv("LISTEN_PID");
+		if (listenpid != NULL) {
+			errno=0;
+			int lpid = strtol(listenpid, NULL, 10);
+			if (errno != 0 || lpid <=0)
+				pexitf("Invalid LISTEN_PID %s", listenpid);
+			if (opt_replace_listen_pid || lpid == getppid()) {
+				gchar *pidstr = g_strdup_printf("%d", getpid());
+				if (!pidstr)
+					pexit("Failed to g_strdup_sprintf pid");
+				if (setenv("LISTEN_PID",pidstr,true) < 0)
+					pexit("Failed to setenv LISTEN_PID");
+				free(pidstr);
+			}
+		}
+
 		execv(g_ptr_array_index(runtime_argv,0), (char **)runtime_argv->pdata);
 		exit(127);
 	}
 
+	if ((signal(SIGTERM, on_sig_exit) == SIG_ERR) || (signal(SIGQUIT, on_sig_exit) == SIG_ERR) || (signal(SIGINT, on_sig_exit) == SIG_ERR))
+		pexit("Failed to register the signal handler");
+
+	if (sigprocmask (SIG_SETMASK, &oldmask, NULL) < 0)
+		pexit("Failed to unblock signals");
+
+	if (opt_exit_command)
+		atexit(do_exit_command);
+
 	g_ptr_array_free (runtime_argv, TRUE);
 
 	/* The runtime has that fd now. We don't need to touch it anymore. */
@@ -1328,18 +1464,18 @@ int main(int argc, char *argv[])
 
 	/* Map pid to its handler.  */
 	GHashTable *pid_to_handler = g_hash_table_new (g_int_hash, g_int_equal);
-	g_hash_table_insert (pid_to_handler, &create_pid, runtime_exit_cb);
+	g_hash_table_insert (pid_to_handler, (pid_t *) &create_pid, runtime_exit_cb);
 
 	/*
 	 * Glib does not support SIGCHLD so use SIGUSR1 with the same semantic.  We will
          * catch SIGCHLD and raise(SIGUSR1) in the signal handler.
 	 */
-	g_unix_signal_add (SIGUSR1, on_sigusr1_cb, pid_to_handler);
+	g_unix_signal_add(SIGUSR1, on_sigusr1_cb, pid_to_handler);
 
 	if (signal(SIGCHLD, on_sigchld) == SIG_ERR)
 		pexit("Failed to set handler for SIGCHLD");
 
-	ninfo("about to waitpid: %d", create_pid);
+	ninfof("about to waitpid: %d", create_pid);
 	if (csname != NULL) {
 		guint terminal_watch = g_unix_fd_add (console_socket_fd, G_IO_IN, terminal_accept_cb, csname);
 		/* Process any SIGCHLD we may have missed before the signal handler was in place.  */
@@ -1356,7 +1492,7 @@ int main(int argc, char *argv[])
 			int old_errno = errno;
 			kill(create_pid, SIGKILL);
 			errno = old_errno;
-			pexit("Failed to wait for `runtime %s`", opt_exec ? "exec" : "create");
+			pexitf("Failed to wait for `runtime %s`", opt_exec ? "exec" : "create");
 		}
 
 	}
@@ -1373,7 +1509,7 @@ int main(int argc, char *argv[])
 				write_sync_fd(sync_pipe_fd, -1, buf);
 			}
 		}
-		nexit("Failed to create container: exit status %d", WEXITSTATUS(runtime_status));
+		nexitf("Failed to create container: exit status %d", WEXITSTATUS(runtime_status));
 	}
 
 	if (opt_terminal && masterfd_stdout == -1)
@@ -1382,15 +1518,15 @@ int main(int argc, char *argv[])
 	/* Read the pid so we can wait for the process to exit */
 	g_file_get_contents(opt_pid_file, &contents, NULL, &err);
 	if (err) {
-		nwarn("Failed to read pidfile: %s", err->message);
+		nwarnf("Failed to read pidfile: %s", err->message);
 		g_error_free(err);
 		exit(1);
 	}
 
 	container_pid = atoi(contents);
-	ninfo("container PID: %d", container_pid);
+	ninfof("container PID: %d", container_pid);
 
-	g_hash_table_insert (pid_to_handler, &container_pid, container_exit_cb);
+	g_hash_table_insert (pid_to_handler, (pid_t *) &container_pid, container_exit_cb);
 
 	/* Setup endpoint for attach */
 	_cleanup_free_ char *attach_symlink_dir_path = NULL;
@@ -1446,13 +1582,15 @@ int main(int argc, char *argv[])
 		exit_status = WEXITSTATUS(container_status);
 	}
 
-	if (!opt_exec) {
+	if (opt_exit_dir) {
 		_cleanup_free_ char *status_str = g_strdup_printf("%d", exit_status);
 		_cleanup_free_ char *exit_file_path = g_build_filename(opt_exit_dir, opt_cid, NULL);
 		if (!g_file_set_contents(exit_file_path, status_str, -1, &err))
-			nexit("Failed to write %s to exit file: %s\n",
+			nexitf("Failed to write %s to exit file: %s",
 			      status_str, err->message);
-	} else {
+	}
+
+	if (opt_exec) {
 		/* Send the command exec exit code back to the parent */
 		write_sync_fd(sync_pipe_fd, exit_status, exit_message);
 	}

contrib/cni/10-crio-bridge.conf

@@ -1,5 +1,5 @@
 {
-    "cniVersion": "0.2.0",
+    "cniVersion": "0.3.0",
     "name": "crio-bridge",
     "type": "bridge",
     "bridge": "cni0",

contrib/cni/99-loopback.conf

@@ -1,4 +1,4 @@
 {
-    "cniVersion": "0.2.0",
+    "cniVersion": "0.3.0",
     "type": "loopback"
 }

contrib/system_containers/centos/config.json.template

@@ -253,6 +253,17 @@
             "source": "/tmp",
             "type": "bind"
         },
+        {
+            "destination": "/var/tmp",
+            "options": [
+                "private",
+                "bind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/var/tmp",
+            "type": "bind"
+        },
         {
             "destination": "/etc",
             "options": [
@@ -385,7 +396,7 @@
             "options": [
                 "rbind",
                 "rprivate",
-                "ro",
+                "rw",
                 "mode=755"
             ],
             "source": "${OPT_CNI}",
@@ -423,5 +434,6 @@
             "source": "/proc",
             "type": "proc"
         }
+        $ADDTL_MOUNTS
     ]
 }

contrib/system_containers/centos/manifest.json

@@ -5,6 +5,7 @@
         "OPT_CNI" : "/opt/cni",
         "VAR_LIB_CONTAINERS_STORAGE" : "/var/lib/containers/storage",
         "VAR_LIB_ORIGIN" : "/var/lib/origin",
-        "VAR_LIB_KUBE" : "/var/lib/kubelet"
+        "VAR_LIB_KUBE" : "/var/lib/kubelet",
+        "ADDTL_MOUNTS" : ""
     }
 }

contrib/system_containers/fedora/config.json.template

@@ -258,6 +258,17 @@
             "source": "/tmp",
             "type": "bind"
         },
+        {
+            "destination": "/var/tmp",
+            "options": [
+                "private",
+                "bind",
+                "rw",
+                "mode=755"
+            ],
+            "source": "/var/tmp",
+            "type": "bind"
+        },
         {
             "destination": "/etc",
             "options": [
@@ -390,7 +401,7 @@
             "options": [
                 "rbind",
                 "rprivate",
-                "ro",
+                "rw",
                 "mode=755"
             ],
             "source": "${OPT_CNI}",
@@ -428,5 +439,6 @@
             "source": "/proc",
             "type": "proc"
         }
+        $ADDTL_MOUNTS
     ]
 }

contrib/system_containers/fedora/manifest.json

@@ -5,6 +5,7 @@
         "OPT_CNI" : "/opt/cni",
         "VAR_LIB_CONTAINERS_STORAGE" : "/var/lib/containers/storage",
         "VAR_LIB_ORIGIN" : "/var/lib/origin",
-        "VAR_LIB_KUBE" : "/var/lib/kubelet"
+        "VAR_LIB_KUBE" : "/var/lib/kubelet",
+        "ADDTL_MOUNTS" : ""
     }
 }

contrib/system_containers/rhel/config.json.template

@@ -248,6 +248,17 @@
       "source": "/tmp",
       "type": "bind"
     },
+    {
+      "destination": "/var/tmp",
+      "options": [
+        "private",
+        "bind",
+        "rw",
+        "mode=755"
+      ],
+      "source": "/var/tmp",
+      "type": "bind"
+    },
     {
       "destination": "/etc",
       "options": [
@@ -380,7 +391,7 @@
       "options": [
         "rbind",
         "rprivate",
-        "ro",
+        "rw",
         "mode=755"
       ],
       "source": "${OPT_CNI}",
@@ -418,5 +429,6 @@
       "source": "/proc",
       "type": "proc"
     }
+    $ADDTL_MOUNTS
   ]
 }

contrib/system_containers/rhel/manifest.json

@@ -5,6 +5,7 @@
     "OPT_CNI": "/opt/cni",
     "VAR_LIB_CONTAINERS_STORAGE": "/var/lib/containers/storage",
     "VAR_LIB_ORIGIN": "/var/lib/origin",
-    "VAR_LIB_KUBE": "/var/lib/kubelet"
+    "VAR_LIB_KUBE": "/var/lib/kubelet",
+    "ADDTL_MOUNTS" : ""
   }
 }

contrib/test/integration/README.md

@@ -1,21 +1,22 @@
-# Fedora and RHEL Integration and End-to-End Tests
+# Fedora and RHEL Test execution
 
-This directory contains playbooks to set up for and run the integration and
-end-to-end tests for CRI-O on RHEL and Fedora hosts. Two entrypoints exist:
+This directory contains playbooks to set up and run, all the CRI-O CI tests
+for both RHEL and Fedora hosts. Two entry-point playbooks exist:
 
- - `main.yml`: sets up the machine and runs tests
- - `results.yml`: gathers test output to `/tmp/artifacts`
+ - `main.yml`: sets up the machine and runs tests.
+ - `results.yml`: gathers test output to `/tmp/artifacts`.
 
-When running `main.yml`, three tags are present:
+When running the `main.yml` playbook, multiple tags are present:
 
- - `setup`: run all tasks to set up the system for testing
- - `e2e`: build CRI-O from source and run Kubernetes node E2Es
- - `integration`: build CRI-O from source and run the local integration suite
+ - `setup`: run all tasks to set up the system for testing.
+ - `e2e`: build CRI-O from source and run Kubernetes end-to-end tests.
+ - `node-e2e`: build CRI-O from source and run Kubernetes 'node' end-to-end tests.
+ - `integration`: build CRI-O from source and run the local integration suite.
 
 The playbooks assume the following things about your system:
 
- - on RHEL, the server and extras repos are configured and certs are present
- - `ansible` is installed and the host is boot-strapped to allow `ansible` to run against it
- - the `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`)
- - CRI-O is checked out to the correct state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o`
- - the user running the playbook has access to passwordless `sudo`
+ - On RHEL, the repositories for EPEL, rhel-server, and extras repos are configured and functional.
+ - The system has been rebooted after installing/updating low-level packages, to ensure they're active.
+ - Ansible is installed, and functional with access to the 'root' user.
+ - The `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`).
+ - The CRI-O repository is present in the desired state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o`.

contrib/test/integration/build/runc.yml

@@ -4,7 +4,7 @@
   git:
     repo: "https://github.com/opencontainers/runc.git"
     dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc"
-    version: "c6e4a1ebeb1a72b529c6f1b6ee2b1ae5b868b14f"
+    version: "ce80fa0a64803d52883955cb77b2708b438a0b28"
 
 - name: build runc
   make:

contrib/test/integration/e2e.yml

@@ -37,6 +37,20 @@
     path: "{{ artifacts }}"
     state: directory
 
+- name: Add repo for new version of git
+  yum_repository:
+    name: Wandisco
+    description: Wandisco git repo
+    baseurl: http://opensource.wandisco.com/rhel/7Server/git/$basearch
+    gpgcheck: no
+  when: ansible_distribution in ['RedHat']
+
+- name: upgrade git package
+  yum:
+    name: 'git'
+    state: latest
+  when: ansible_distribution in ['RedHat']
+
 # TODO remove the last test skipped once https://github.com/kubernetes-incubator/cri-o/pull/1217 is merged
 - name: Buffer the e2e testing command to workaround Ansible YAML folding "feature"
   set_fact:
@@ -44,7 +58,7 @@
         /usr/bin/go run hack/e2e.go
             --test
             --test_args="-host=https://{{ ansible_default_ipv4.address }}:6443
-                        --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|PersistentVolumes|\[HPA\]|should.support.building.a.client.with.a.CSR|should.support.inline.execution.and.attach
+                        --ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|PersistentVolumes|\[HPA\]|should.support.building.a.client.with.a.CSR|should.support.inline.execution.and.attach|should.propagate.mounts.to.the.host
                         --report-dir={{ artifacts }}"
             &> {{ artifacts }}/e2e.log
   # Fix vim syntax hilighting: "

contrib/test/integration/main.yml

@@ -30,6 +30,7 @@
 
     - name: clone build and install runc
       include: "build/runc.yml"
+      when: "{{ build_runc | default(True) | bool}}"
 
     - name: clone build and install networking plugins
       include: "build/plugins.yml"
@@ -58,7 +59,7 @@
       include: "build/cri-tools.yml"
       vars:
         force_clone: True
-        cri_tools_git_version: "a9e38a4a000bc1a4052fb33de1c967b8cfe9ad40"
+        cri_tools_git_version: "240a840375cdabb5860c75c99e8b0d0a776006b4"
     - name: run cri-o integration tests
       include: test.yml
 
@@ -77,7 +78,7 @@
       include: "build/cri-tools.yml"
       vars:
           force_clone: True
-          cri_tools_git_version: "a9e38a4a000bc1a4052fb33de1c967b8cfe9ad40"
+          cri_tools_git_version: "240a840375cdabb5860c75c99e8b0d0a776006b4"
     - name: run critest validation and benchmarks
       include: critest.yml
 
@@ -117,9 +118,8 @@
       include: "build/kubernetes.yml"
       vars:
           force_clone: True
-          # master as of 12/11/2017
-          k8s_git_version: "master-nfs-fix"
-          k8s_github_fork: "runcom"
+          k8s_git_version: "release-1.10"
+          k8s_github_fork: "kubernetes"
           crio_socket: "/var/run/crio/crio.sock"
     - name: run k8s e2e tests
       include: e2e.yml

contrib/test/integration/system.yml

@@ -55,7 +55,7 @@
     - socat
     - tar
     - wget
-  async: 600
+  async: '{{ 20 * 60 }}'
   poll: 10
 
 - name: Add python2-boto for Fedora

docs/crio.8.md

@@ -94,8 +94,6 @@ crio [GLOBAL OPTIONS] config [OPTIONS]
 
 **--pids-limit**="": Maximum number of processes allowed in a container (default: 1024)
 
-**--enable-shared-pid-namespace**="": Enable using a shared PID namespace for containers in a pod (default: false)
-
 **--root**="": The crio root dir (default: "/var/lib/containers/storage")
 
 **--registry**="": Registry host which will be prepended to unqualified images, can be specified multiple times

docs/crio.conf.5.md

@@ -87,9 +87,6 @@ Example:
 **pids_limit**=""
   Maximum number of processes allowed in a container (default: 1024)
 
-**enable_shared_pid_namespace**=""
-  Enable using a shared PID namespace for containers in a pod (default: false)
-
 **runtime**=""
   OCI runtime path (default: "/usr/bin/runc")
 

hooks.md

@@ -1,76 +1,44 @@
 # OCI Hooks Configuration
 
-[The OCI Runtime Specification defines POSIX-platform Hooks:](
-https://github.com/opencontainers/runtime-spec/blob/master/config.md#posix-platform-hooks)
+For POSIX platforms, the [OCI runtime configuration][runtime-spec] supports [hooks][spec-hooks] for configuring custom actions related to the life cycle of the container.
+The way you enable the hooks above is by editing the OCI runtime configuration before running the OCI runtime (e.g. [`runc`][runc]).
+CRI-O and `podman create` create the OCI configuration for you, and this documentation allows developers to configure CRI-O to set their intended hooks.
 
-## POSIX-platform Hooks
+One problem with hooks is that the runtime actually stalls execution of the container before running the hooks and stalls completion of the container, until all hooks complete.  This can cause some performance issues.  Also a lot of hooks just check if certain configuration is set and then exit early, without doing anything.  For example the [oci-systemd-hook](https://github.com/projectatomic/oci-systemd-hook) only executes if the command is `init` or `systemd`, otherwise it just exits.  This means if we automatically enabled all hooks, every container would have to execute `oci-systemd-hook`, even if they don't run systemd inside of the container.   Performance would also suffer if we exectuted each hook at each stage ([pre-start][], [post-start][], and [post-stop][]).
 
-For POSIX platforms, the configuration structure supports hooks for configuring custom actions related to the life cycle of the container.
+## Notational Conventions
 
-hooks (object, OPTIONAL) MAY contain any of the following properties:
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and "OPTIONAL" are to be interpreted as described in [RFC 2119][rfc2119].
 
- *  prestart (array of objects, OPTIONAL) is an array of pre-start hooks. Entries in the array contain the following properties:
-    * path (string, REQUIRED) with similar semantics to [IEEE Std 1003.1-2008 execv's path][ieee-1003.1-2008-functions-exec]. This specification extends the IEEE standard in that path MUST be absolute.
-    * args (array of strings, OPTIONAL) with the same semantics as [IEEE Std 1003.1-2008 execv's argv][ieee-1003.1-2008-functions-exec].
-    * env (array of strings, OPTIONAL) with the same semantics as IEEE Std 1003.1-2008's environ.
-    * timeout (int, OPTIONAL) is the number of seconds before aborting the hook. If set, timeout MUST be greater than zero.
- * poststart (array of objects, OPTIONAL) is an array of post-start hooks. Entries in the array have the same schema as pre-start entries.
- * poststop (array of objects, OPTIONAL) is an array of post-stop hooks. Entries in the array have the same schema as pre-start entries.
+## JSON Definition
 
-Hooks allow users to specify programs to run before or after various lifecycle events. Hooks MUST be called in the listed order. The state of the container MUST be passed to hooks over stdin so that they may do work appropriate to the current state of the container.
+CRI-O reads all [JSON][] files in `/usr/share/containers/oci/hooks.d/*.json` and `/etc/containers/oci/hooks.d/*.json` to load hook configuration.
+If the same file is in both directories, the one in `/etc/containers/oci/hooks.d` takes precedence.
 
-### Prestart
+Each JSON file should contain an object with the following properties:
 
-The Prestart hooks MUST be called after the start operation is called but before the user-specified program command is executed. On Linux, for example, they are called after the container namespaces are created, so they provide an opportunity to customize the container (e.g. the network namespace could be specified in this hook).
+* **`hook`** (REQUIRED, string) Sets [`path`][spec-hooks] in the injected hook.
+* **`arguments`** (OPTIONAL, array of strings) Additional arguments to pass to the hook.
+    The injected hook's [`args`][spec-hooks] is `hook` with `arguments` appended.
+* **`stages`** (REQUIRED, array of strings) Stages when the hook MUST be injected.
+    Entries MUST be chosen from:
+    * **`prestart`**, to inject [pre-start][].
+    * **`poststart`**, to inject [post-start][].
+    * **`poststop`**, to inject [post-stop][].
+* **`cmds`** (OPTIONAL, array of strings) The hook MUST be injected if the configured [`process.args[0]`][spec-process] matches an entry.
+    Entries MUST be [POSIX extended regular expressions][POSIX-ERE].
+* **`annotations`** (OPTIONAL, array of strings) The hook MUST be injected if the configured [`annotations`][spec-annotations] matches an entry.
+    Entries MUST be [POSIX extended regular expressions][POSIX-ERE].
+* **`hasbindmounts`** (OPTIONAL, boolean) The hook MUST be injected if `hasbindmounts` is true and the container is configured to bind-mount host directories into the container.
 
-### Poststart
+The matching properties (`cmds`, `annotations` and `hasbindmounts`) are orthogonal, and the hook is injected if *any* of those properties match.
 
-The post-start hooks MUST be called after the user-specified process is executed but before the start operation returns. For example, this hook can notify the user that the container process is spawned.
+## Example
 
-### Poststop
+The following configuration tells CRI-O to inject `oci-systemd-hook` in the [pre-start][] and [post-stop][] stages if [`process.args[0]`][spec-process] ends with `/init` or `/systemd`:
 
-The post-stop hooks MUST be called after the container is deleted but before the delete operation returns. Cleanup or debugging functions are examples of such a hook.
-
-## CRI-O configuration files for automatically enabling Hooks
-
-The way you enable the hooks above is by editing the OCI Specification to add your hook before running the oci runtime, like runc.  But this is what `CRI-O` and `Kpod create` do for you, so we wanted a way for developers to drop configuration files onto the system, so that their hooks would be able to be plugged in.
-
-One problem with hooks is that the runtime actually stalls execution of the container before running the hooks and stalls completion of the container, until all hooks complete.  This can cause some performance issues.  Also a lot of hooks just check if certain configuration is set and then exit early, without doing anything.  For example the [oci-systemd-hook](https://github.com/projectatomic/oci-systemd-hook) only executes if the command is `init` or `systemd`, otherwise it just exits.  This means if we automatically enable all hooks, every container will have to execute oci-systemd-hook, even if they don't run systemd inside of the container.   Also since there are three stages, prestart, poststart, poststop each hook gets executed three times.
-
-
-
-### Json Definition
-
-We decided to add a json file for hook builders which allows them to tell CRI-O when to run the hook and in which stage.
-CRI-O reads all json files in /usr/share/containers/oci/hooks.d/*.json and /etc/containers/oci/hooks.d and sets up the specified hooks to run.  If the same name is in both directories, the one in /etc/containers/oci/hooks.d takes precedence.
-
-The json configuration looks like this in GO
-```
-// HookParams is the structure returned from read the hooks configuration
-type HookParams struct {
-	Hook          string   `json:"hook"`
-	Stage         []string `json:"stages"`
-	Cmds          []string `json:"cmds"`
-	Annotations   []string `json:"annotations"`
-	HasBindMounts bool     `json:"hasbindmounts"`
-	Arguments     []string `json:"arguments"`
-}
-```
-
-| Key    | Description                                                                                                                        | Required/Optional |
-| ------ |----------------------------------------------------------------------------------------------------------------------------------- | -------- |
-| hook   | Path to the hook                                                                                                                   | Required |
-| stages | List of stages to run the hook in: Valid options are `prestart`, `poststart`, `poststop`                                           | Required |
-| cmds   | List of regular expressions to match the command for running the container.  If the command matches a regex, the hook will be run  | Optional |
-| annotations | List of regular expressions to match against the Annotations in the container runtime spec, if an Annotation matches the hook will be run|optional |
-| hasbindmounts | Tells CRI-O to run the hook if the container has bind mounts from the host into the container | Optional |
-| arguments | Additional arguments to append to the hook command when executing it. For example --debug | Optional |
-
-### Example
-
-
-```
-cat /etc/containers/oci/hooks.d/oci-systemd-hook.json
+```console
+$ cat /etc/containers/oci/hooks.d/oci-systemd-hook.json
 {
     "cmds": [".*/init$" , ".*/systemd$" ],
     "hook": "/usr/libexec/oci/hooks.d/oci-systemd-hook",
@@ -78,10 +46,9 @@ cat /etc/containers/oci/hooks.d/oci-systemd-hook.json
 }
 ```
 
-In the above example CRI-O will only run the oci-systemd-hook in the prestart and poststop stage, if the command ends with /init or /systemd
+The following example tells CRI-O to inject `oci-umount --debug` in the [pre-start][] phase if the container is configured to bind-mount host directories into the container.
 
-
-```
+```console
 cat /etc/containers/oci/hooks.d/oci-systemd-hook.json
 {
     "hasbindmounts": true,
@@ -90,4 +57,15 @@ cat /etc/containers/oci/hooks.d/oci-systemd-hook.json
     "arguments": [ "--debug" ]
 }
 ```
-In this example the oci-umount will only be run during the prestart phase if the container has volume/bind mounts from the host into the container, it will also execute oci-umount with the --debug argument.
+
+[JSON]: https://tools.ietf.org/html/rfc8259
+[POSIX-ERE]: http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04
+[post-start]: https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#poststart
+[post-stop]: https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#poststop
+[pre-start]: https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#prestart
+[rfc2119]: http://tools.ietf.org/html/rfc2119
+[runc]: https://github.com/opencontainers/runc
+[runtime-spec]: https://github.com/opencontainers/runtime-spec/blob/v1.0.1/spec.md
+[spec-annotations]: https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#annotations
+[spec-hooks]: https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
+[spec-process]: https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#process

lib/config.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/kubernetes-incubator/cri-o/oci"
-	"github.com/opencontainers/selinux/go-selinux"
 )
 
 // Default paths if none are specified
@@ -121,9 +120,6 @@ type RuntimeConfig struct {
 	// NoPivot instructs the runtime to not use `pivot_root`, but instead use `MS_MOVE`
 	NoPivot bool `toml:"no_pivot"`
 
-	// EnableSharePidNamespace instructs the runtime to enable share pid namespace
-	EnableSharedPIDNamespace bool `toml:"enable_shared_pid_namespace"`
-
 	// Conmon is the path to conmon binary, used for managing the runtime.
 	Conmon string `toml:"conmon"`
 
@@ -287,7 +283,7 @@ func DefaultConfig() *Config {
 			ConmonEnv: []string{
 				"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 			},
-			SELinux:           selinux.GetEnabled(),
+			SELinux:           selinuxEnabled(),
 			SeccompProfile:    seccompProfilePath,
 			ApparmorProfile:   apparmorProfileName,
 			CgroupManager:     cgroupManager,

lib/config_linux.go

@@ -0,0 +1,9 @@
+// +build linux
+
+package lib
+
+import selinux "github.com/opencontainers/selinux/go-selinux"
+
+func selinuxEnabled() bool {
+	return selinux.GetEnabled()
+}

lib/config_unsupported.go

@@ -0,0 +1,7 @@
+// +build !linux
+
+package lib
+
+func selinuxEnabled() bool {
+	return false
+}

lib/container_server.go

@@ -17,13 +17,12 @@ import (
 	"github.com/kubernetes-incubator/cri-o/pkg/annotations"
 	"github.com/kubernetes-incubator/cri-o/pkg/registrar"
 	"github.com/kubernetes-incubator/cri-o/pkg/storage"
-	"github.com/opencontainers/runc/libcontainer"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+	"k8s.io/kubernetes/pkg/kubelet/network/hostport"
 )
 
 // ContainerServer implements the ImageServer
@@ -38,6 +37,7 @@ type ContainerServer struct {
 	podNameIndex         *registrar.Registrar
 	podIDIndex           *truncindex.TruncIndex
 	hooks                map[string]HookParams
+	hooksLock            sync.Mutex
 
 	imageContext *types.SystemContext
 	stateLock    sync.Locker
@@ -52,7 +52,41 @@ func (c *ContainerServer) Runtime() *oci.Runtime {
 
 // Hooks returns the oci hooks for the ContainerServer
 func (c *ContainerServer) Hooks() map[string]HookParams {
-	return c.hooks
+	hooks := map[string]HookParams{}
+	c.hooksLock.Lock()
+	defer c.hooksLock.Unlock()
+	for key, h := range c.hooks {
+		hooks[key] = h
+	}
+	return hooks
+}
+
+// RemoveHook removes an hook by name
+func (c *ContainerServer) RemoveHook(hook string) {
+	c.hooksLock.Lock()
+	defer c.hooksLock.Unlock()
+	if _, ok := c.hooks[hook]; ok {
+		delete(c.hooks, hook)
+	}
+}
+
+// AddHook adds an hook by hook's path
+func (c *ContainerServer) AddHook(hookPath string) {
+	c.hooksLock.Lock()
+	defer c.hooksLock.Unlock()
+	hook, err := readHook(hookPath)
+	if err != nil {
+		logrus.Debugf("error while reading hook %s", hookPath)
+		return
+	}
+	for key, h := range c.hooks {
+		// hook.Hook can only be defined in one hook file, unless it has the
+		// same name in the override path.
+		if hook.Hook == h.Hook && key != filepath.Base(hookPath) {
+			logrus.Debugf("duplicate path,  hook %q from %q already defined in %q", hook.Hook, c.config.HooksDirPath, key)
+		}
+	}
+	c.hooks[filepath.Base(hookPath)] = hook
 }
 
 // Store returns the Store for the ContainerServer
@@ -152,6 +186,7 @@ func New(config *Config) (*ContainerServer, error) {
 			}
 		}
 	}
+	logrus.Debugf("hooks %+v", hooks)
 
 	return &ContainerServer{
 		runtime:              runtime,
@@ -332,16 +367,26 @@ func (c *ContainerServer) LoadSandbox(id string) error {
 		return err
 	}
 
+	portMappings := []*hostport.PortMapping{}
+	if err := json.Unmarshal([]byte(m.Annotations[annotations.PortMappings]), &portMappings); err != nil {
+		return err
+	}
+
 	privileged := isTrue(m.Annotations[annotations.PrivilegedRuntime])
 	trusted := isTrue(m.Annotations[annotations.TrustedSandbox])
+	nsOpts := pb.NamespaceOption{}
+	if err := json.Unmarshal([]byte(m.Annotations[annotations.NamespaceOptions]), &nsOpts); err != nil {
+		return err
+	}
 
-	sb, err := sandbox.New(id, name, m.Annotations[annotations.KubeName], filepath.Dir(m.Annotations[annotations.LogPath]), "", labels, kubeAnnotations, processLabel, mountLabel, &metadata, m.Annotations[annotations.ShmPath], "", privileged, trusted, m.Annotations[annotations.ResolvPath], "", nil)
+	sb, err := sandbox.New(id, m.Annotations[annotations.Namespace], name, m.Annotations[annotations.KubeName], filepath.Dir(m.Annotations[annotations.LogPath]), labels, kubeAnnotations, processLabel, mountLabel, &metadata, m.Annotations[annotations.ShmPath], m.Annotations[annotations.CgroupParent], privileged, trusted, m.Annotations[annotations.ResolvPath], m.Annotations[annotations.HostName], portMappings)
 	if err != nil {
 		return err
 	}
 	sb.AddHostnamePath(m.Annotations[annotations.HostnamePath])
 	sb.AddIP(ip)
 	sb.SetSeccompProfilePath(spp)
+	sb.SetNamespaceOptions(&nsOpts)
 
 	// We add a netNS only if we can load a permanent one.
 	// Otherwise, the sandbox will live in the host namespace.
@@ -389,7 +434,7 @@ func (c *ContainerServer) LoadSandbox(id string) error {
 		return err
 	}
 
-	scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
+	scontainer, err := oci.NewContainer(m.Annotations[annotations.ContainerID], cname, sandboxPath, m.Annotations[annotations.LogPath], sb.NetNs().Path(), labels, m.Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, privileged, trusted, sandboxDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
 	if err != nil {
 		return err
 	}
@@ -513,7 +558,7 @@ func (c *ContainerServer) LoadContainer(id string) error {
 		return err
 	}
 
-	ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs(), labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
+	ctr, err := oci.NewContainer(id, name, containerPath, m.Annotations[annotations.LogPath], sb.NetNs().Path(), labels, m.Annotations, kubeAnnotations, img, imgName, imgRef, &metadata, sb.ID(), tty, stdin, stdinOnce, sb.Privileged(), sb.Trusted(), containerDir, created, m.Annotations["org.opencontainers.image.stopSignal"])
 	if err != nil {
 		return err
 	}
@@ -690,7 +735,7 @@ func (c *ContainerServer) AddSandbox(sb *sandbox.Sandbox) {
 	c.state.sandboxes.Add(sb.ID(), sb)
 
 	c.stateLock.Lock()
-	c.state.processLevels[selinux.NewContext(sb.ProcessLabel())["level"]]++
+	c.addSandboxPlatform(sb)
 	c.stateLock.Unlock()
 }
 
@@ -713,18 +758,9 @@ func (c *ContainerServer) HasSandbox(id string) bool {
 // RemoveSandbox removes a sandbox from the state store
 func (c *ContainerServer) RemoveSandbox(id string) {
 	sb := c.state.sandboxes.Get(id)
-	processLabel := sb.ProcessLabel()
-	level := selinux.NewContext(processLabel)["level"]
 
 	c.stateLock.Lock()
-	pl, ok := c.state.processLevels[level]
-	if ok {
-		c.state.processLevels[level] = pl - 1
-		if c.state.processLevels[level] == 0 {
-			label.ReleaseLabel(processLabel)
-			delete(c.state.processLevels, level)
-		}
-	}
+	c.removeSandboxPlatform(sb)
 	c.stateLock.Unlock()
 
 	c.state.sandboxes.Delete(id)
@@ -734,20 +770,3 @@ func (c *ContainerServer) RemoveSandbox(id string) {
 func (c *ContainerServer) ListSandboxes() []*sandbox.Sandbox {
 	return c.state.sandboxes.List()
 }
-
-// LibcontainerStats gets the stats for the container with the given id from runc/libcontainer
-func (c *ContainerServer) LibcontainerStats(ctr *oci.Container) (*libcontainer.Stats, error) {
-	// TODO: make this not hardcoded
-	// was: c.runtime.Path(ociContainer) but that returns /usr/bin/runc - how do we get /run/runc?
-	// runroot is /var/run/runc
-	// Hardcoding probably breaks ClearContainers compatibility
-	factory, err := loadFactory("/run/runc")
-	if err != nil {
-		return nil, err
-	}
-	container, err := factory.Load(ctr.ID())
-	if err != nil {
-		return nil, err
-	}
-	return container.Stats()
-}

lib/container_server_linux.go

@@ -0,0 +1,80 @@
+// +build linux
+
+package lib
+
+import (
+	"path/filepath"
+	"time"
+
+	"github.com/kubernetes-incubator/cri-o/lib/sandbox"
+	"github.com/kubernetes-incubator/cri-o/oci"
+	"github.com/opencontainers/runc/libcontainer"
+	selinux "github.com/opencontainers/selinux/go-selinux"
+	"github.com/opencontainers/selinux/go-selinux/label"
+)
+
+// libcontainerStats gets the stats for the container with the given id from runc/libcontainer
+func (c *ContainerServer) libcontainerStats(ctr *oci.Container) (*libcontainer.Stats, error) {
+	// TODO: make this not hardcoded
+	// was: c.runtime.Path(ociContainer) but that returns /usr/bin/runc - how do we get /run/runc?
+	// runroot is /var/run/runc
+	// Hardcoding probably breaks ClearContainers compatibility
+	factory, err := loadFactory("/run/runc")
+	if err != nil {
+		return nil, err
+	}
+	container, err := factory.Load(ctr.ID())
+	if err != nil {
+		return nil, err
+	}
+	return container.Stats()
+}
+
+func (c *ContainerServer) addSandboxPlatform(sb *sandbox.Sandbox) {
+	c.state.processLevels[selinux.NewContext(sb.ProcessLabel())["level"]]++
+}
+
+func (c *ContainerServer) removeSandboxPlatform(sb *sandbox.Sandbox) {
+	processLabel := sb.ProcessLabel()
+	level := selinux.NewContext(processLabel)["level"]
+	pl, ok := c.state.processLevels[level]
+	if ok {
+		c.state.processLevels[level] = pl - 1
+		if c.state.processLevels[level] == 0 {
+			label.ReleaseLabel(processLabel)
+			delete(c.state.processLevels, level)
+		}
+	}
+}
+
+func loadFactory(root string) (libcontainer.Factory, error) {
+	abs, err := filepath.Abs(root)
+	if err != nil {
+		return nil, err
+	}
+	cgroupManager := libcontainer.Cgroupfs
+	return libcontainer.New(abs, cgroupManager, libcontainer.CriuPath(""))
+}
+
+func (c *ContainerServer) getContainerStats(ctr *oci.Container, previousStats *ContainerStats) (*ContainerStats, error) {
+	previousCPU := previousStats.CPUNano
+	previousSystem := previousStats.SystemNano
+	libcontainerStats, err := c.libcontainerStats(ctr)
+	if err != nil {
+		return nil, err
+	}
+	cgroupStats := libcontainerStats.CgroupStats
+	stats := new(ContainerStats)
+	stats.Container = ctr.ID()
+	stats.CPUNano = cgroupStats.CpuStats.CpuUsage.TotalUsage
+	stats.SystemNano = time.Now().UnixNano()
+	stats.CPU = calculateCPUPercent(libcontainerStats, previousCPU, previousSystem)
+	stats.MemUsage = cgroupStats.MemoryStats.Usage.Usage
+	stats.MemLimit = getMemLimit(cgroupStats.MemoryStats.Usage.Limit)
+	stats.MemPerc = float64(stats.MemUsage) / float64(stats.MemLimit)
+	stats.PIDs = cgroupStats.PidsStats.Current
+	stats.BlockInput, stats.BlockOutput = calculateBlockIO(libcontainerStats)
+	stats.NetInput, stats.NetOutput = getContainerNetIO(libcontainerStats)
+
+	return stats, nil
+}

lib/container_server_unsupported.go

@@ -0,0 +1,22 @@
+// +build !linux
+
+package lib
+
+import (
+	"github.com/kubernetes-incubator/cri-o/lib/sandbox"
+	"github.com/kubernetes-incubator/cri-o/oci"
+	"github.com/pkg/errors"
+)
+
+func (c *ContainerServer) addSandboxPlatform(sb *sandbox.Sandbox) {
+	// nothin' doin'
+}
+
+func (c *ContainerServer) removeSandboxPlatform(sb *sandbox.Sandbox) {
+	// nothin' doin'
+}
+
+func (c *ContainerServer) getContainerStats(ctr *oci.Container, previousStats *ContainerStats) (*ContainerStats, error) {
+	// nothin' doin'
+	return nil, errors.New("container stats not supported")
+}

lib/hooks.go

@@ -30,9 +30,16 @@ type HookParams struct {
 	Arguments     []string `json:"arguments"`
 }
 
+var (
+	errNotJSON = errors.New("hook file isn't a JSON")
+)
+
 // readHook reads hooks json files, verifies it and returns the json config
 func readHook(hookPath string) (HookParams, error) {
 	var hook HookParams
+	if !strings.HasSuffix(hookPath, ".json") {
+		return hook, errNotJSON
+	}
 	raw, err := ioutil.ReadFile(hookPath)
 	if err != nil {
 		return hook, errors.Wrapf(err, "error Reading hook %q", hookPath)
@@ -54,9 +61,13 @@ func readHook(hookPath string) (HookParams, error) {
 			return hook, errors.Wrapf(err, "invalid cmd regular expression %q defined in hook config %q", cmd, hookPath)
 		}
 	}
-	for _, stage := range hook.Stage {
-		if !validStage[stage] {
-			return hook, errors.Wrapf(err, "unknown stage %q defined in hook config %q", stage, hookPath)
+	if len(hook.Stage) == 0 {
+		logrus.Warnf("No stage defined in hook config %q, hook will never run", hookPath)
+	} else {
+		for _, stage := range hook.Stage {
+			if !validStage[stage] {
+				return hook, errors.Wrapf(err, "unknown stage %q defined in hook config %q", stage, hookPath)
+			}
 		}
 	}
 	return hook, nil
@@ -79,11 +90,11 @@ func readHooks(hooksPath string, hooks map[string]HookParams) error {
 	}
 
 	for _, file := range files {
-		if !strings.HasSuffix(file.Name(), ".json") {
-			continue
-		}
 		hook, err := readHook(filepath.Join(hooksPath, file.Name()))
 		if err != nil {
+			if err == errNotJSON {
+				continue
+			}
 			return err
 		}
 		for key, h := range hooks {

lib/logs.go

@@ -8,7 +8,7 @@ import (
 	"github.com/hpcloud/tail"
 )
 
-// LogOptions contains all of the options for displaying logs in kpod
+// LogOptions contains all of the options for displaying logs in podman
 type LogOptions struct {
 	Details   bool
 	Follow    bool

lib/rename.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"path/filepath"
 
-	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/kubernetes-incubator/cri-o/oci"

lib/sandbox/sandbox.go

@@ -1,70 +1,18 @@
 package sandbox
 
 import (
-	"crypto/rand"
 	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
-	"sync"
 	"time"
 
-	"github.com/containernetworking/plugins/pkg/ns"
-	"github.com/docker/docker/pkg/mount"
-	"github.com/docker/docker/pkg/symlink"
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
-	"golang.org/x/sys/unix"
 	"k8s.io/apimachinery/pkg/fields"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	"k8s.io/kubernetes/pkg/kubelet/network/hostport"
 )
 
-// NetNs handles data pertaining a network namespace
-type NetNs struct {
-	sync.Mutex
-	ns       ns.NetNS
-	symlink  *os.File
-	closed   bool
-	restored bool
-}
-
-func (ns *NetNs) symlinkCreate(name string) error {
-	b := make([]byte, 4)
-	_, randErr := rand.Reader.Read(b)
-	if randErr != nil {
-		return randErr
-	}
-
-	nsName := fmt.Sprintf("%s-%x", name, b)
-	symlinkPath := filepath.Join(NsRunDir, nsName)
-
-	if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil {
-		return err
-	}
-
-	fd, err := os.Open(symlinkPath)
-	if err != nil {
-		if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
-			return removeErr
-		}
-
-		return err
-	}
-
-	ns.symlink = fd
-
-	return nil
-}
-
-func (ns *NetNs) symlinkRemove() error {
-	if err := ns.symlink.Close(); err != nil {
-		return err
-	}
-
-	return os.RemoveAll(ns.symlink.Name())
-}
-
 func isSymbolicLink(path string) (bool, error) {
 	fi, err := os.Lstat(path)
 	if err != nil {
@@ -76,7 +24,7 @@ func isSymbolicLink(path string) (bool, error) {
 
 // NetNsGet returns the NetNs associated with the given nspath and name
 func NetNsGet(nspath, name string) (*NetNs, error) {
-	if err := ns.IsNSorErr(nspath); err != nil {
+	if err := isNSorErr(nspath); err != nil {
 		return nil, ErrClosedNetNS
 	}
 
@@ -96,13 +44,11 @@ func NetNsGet(nspath, name string) (*NetNs, error) {
 		resolvedNsPath = nspath
 	}
 
-	netNS, err := ns.GetNS(resolvedNsPath)
+	netNs, err := getNetNs(resolvedNsPath)
 	if err != nil {
 		return nil, err
 	}
 
-	netNs := &NetNs{ns: netNS, closed: false, restored: true}
-
 	if symlink {
 		fd, err := os.Open(nspath)
 		if err != nil {
@@ -121,13 +67,7 @@ func NetNsGet(nspath, name string) (*NetNs, error) {
 
 // HostNetNsPath returns the current network namespace for the host
 func HostNetNsPath() (string, error) {
-	netNS, err := ns.GetCurrentNS()
-	if err != nil {
-		return "", err
-	}
-
-	defer netNS.Close()
-	return netNS.Path(), nil
+	return hostNetNsPath()
 }
 
 // Sandbox contains data surrounding kubernetes sandboxes on the server
@@ -147,6 +87,7 @@ type Sandbox struct {
 	mountLabel     string
 	netns          *NetNs
 	metadata       *pb.PodSandboxMetadata
+	nsOpts         *pb.NamespaceOption
 	shmPath        string
 	cgroupParent   string
 	privileged     bool
@@ -160,6 +101,7 @@ type Sandbox struct {
 	ip                 string
 	seccompProfilePath string
 	created            time.Time
+	hostNetwork        bool
 }
 
 const (
@@ -224,6 +166,16 @@ func (s *Sandbox) AddIP(ip string) {
 	s.ip = ip
 }
 
+// SetNamespaceOptions sets whether the pod is running using host network
+func (s *Sandbox) SetNamespaceOptions(nsOpts *pb.NamespaceOption) {
+	s.nsOpts = nsOpts
+}
+
+// NamespaceOptions returns the namespace options for the sandbox
+func (s *Sandbox) NamespaceOptions() *pb.NamespaceOption {
+	return s.nsOpts
+}
+
 // IP returns the ip of the sandbox
 func (s *Sandbox) IP() string {
 	return s.ip
@@ -372,18 +324,14 @@ func (s *Sandbox) RemoveInfraContainer() {
 
 // NetNs retrieves the network namespace of the sandbox
 // If the sandbox uses the host namespace, nil is returned
-func (s *Sandbox) NetNs() ns.NetNS {
-	if s.netns == nil {
-		return nil
-	}
-
-	return s.netns.ns
+func (s *Sandbox) NetNs() *NetNs {
+	return s.netns
 }
 
 // NetNsPath returns the path to the network namespace of the sandbox.
 // If the sandbox uses the host namespace, nil is returned
 func (s *Sandbox) NetNsPath() string {
-	if s.netns == nil {
+	if s.netns == nil || s.netns.symlink == nil {
 		return ""
 	}
 
@@ -396,20 +344,16 @@ func (s *Sandbox) NetNsCreate() error {
 		return fmt.Errorf("net NS already created")
 	}
 
-	netNS, err := ns.NewNS()
+	netNS, err := newNetNs()
 	if err != nil {
 		return err
 	}
-
-	s.netns = &NetNs{
-		ns:     netNS,
-		closed: false,
-	}
+	s.netns = netNS
 
 	if err := s.netns.symlinkCreate(s.name); err != nil {
 		logrus.Warnf("Could not create nentns symlink %v", err)
 
-		if err1 := s.netns.ns.Close(); err1 != nil {
+		if err1 := s.netns.Close(); err1 != nil {
 			return err1
 		}
 
@@ -456,43 +400,5 @@ func (s *Sandbox) NetNsRemove() error {
 		return nil
 	}
 
-	s.netns.Lock()
-	defer s.netns.Unlock()
-
-	if s.netns.closed {
-		// netNsRemove() can be called multiple
-		// times without returning an error.
-		return nil
-	}
-
-	if err := s.netns.symlinkRemove(); err != nil {
-		return err
-	}
-
-	if err := s.netns.ns.Close(); err != nil {
-		return err
-	}
-
-	if s.netns.restored {
-		// we got namespaces in the form of
-		// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
-		// but /var/run on most system is symlinked to /run so we first resolve
-		// the symlink and then try and see if it's mounted
-		fp, err := symlink.FollowSymlinkInScope(s.netns.ns.Path(), "/")
-		if err != nil {
-			return err
-		}
-		if mounted, err := mount.Mounted(fp); err == nil && mounted {
-			if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
-				return err
-			}
-		}
-
-		if err := os.RemoveAll(s.netns.ns.Path()); err != nil {
-			return err
-		}
-	}
-
-	s.netns.closed = true
-	return nil
+	return s.netns.Remove()
 }

lib/sandbox/sandbox_linux.go

@@ -0,0 +1,151 @@
+// +build linux
+
+package sandbox
+
+import (
+	"crypto/rand"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/docker/pkg/symlink"
+	"golang.org/x/sys/unix"
+)
+
+func isNSorErr(nspath string) error {
+	return ns.IsNSorErr(nspath)
+}
+
+func newNetNs() (*NetNs, error) {
+	netNS, err := ns.NewNS()
+	if err != nil {
+		return nil, err
+	}
+
+	return &NetNs{nn: netNS, closed: false}, nil
+}
+
+func getNetNs(path string) (*NetNs, error) {
+	return &NetNs{}, nil
+	netNS, err := ns.GetNS(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &NetNs{nn: netNS, closed: false, restored: true}, nil
+}
+
+// NetNs handles data pertaining a network namespace
+type NetNs struct {
+	sync.Mutex
+	nn       ns.NetNS
+	symlink  *os.File
+	closed   bool
+	restored bool
+}
+
+func (nns *NetNs) Path() string {
+	return nns.nn.Path()
+}
+
+func (nns *NetNs) Close() error {
+	return nns.nn.Close()
+}
+
+func (nns *NetNs) Remove() error {
+	nns.Lock()
+	defer nns.Unlock()
+
+	if nns.closed {
+		// netNsRemove() can be called multiple
+		// times without returning an error.
+		return nil
+	}
+
+	if err := nns.symlinkRemove(); err != nil {
+		return err
+	}
+
+	if err := nns.Close(); err != nil {
+		return err
+	}
+
+	nns.closed = true
+
+	if nns.restored {
+		// we got namespaces in the form of
+		// /var/run/netns/cni-0d08effa-06eb-a963-f51a-e2b0eceffc5d
+		// but /var/run on most system is symlinked to /run so we first resolve
+		// the symlink and then try and see if it's mounted
+		fp, err := symlink.FollowSymlinkInScope(nns.Path(), "/")
+		if err != nil {
+			return err
+		}
+		if mounted, err := mount.Mounted(fp); err == nil && mounted {
+			if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
+				return err
+			}
+		}
+
+		if nns.Path() != "" {
+			if err := os.RemoveAll(nns.Path()); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (nns *NetNs) symlinkCreate(name string) error {
+	b := make([]byte, 4)
+	_, randErr := rand.Reader.Read(b)
+	if randErr != nil {
+		return randErr
+	}
+
+	nsName := fmt.Sprintf("%s-%x", name, b)
+	symlinkPath := filepath.Join(NsRunDir, nsName)
+
+	if err := os.Symlink(nns.nn.Path(), symlinkPath); err != nil {
+		return err
+	}
+
+	fd, err := os.Open(symlinkPath)
+	if err != nil {
+		if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
+			return removeErr
+		}
+
+		return err
+	}
+
+	nns.symlink = fd
+
+	return nil
+}
+
+func (nns *NetNs) symlinkRemove() error {
+	if err := nns.symlink.Close(); err != nil {
+		return fmt.Errorf("failed to close net ns symlink: %v", err)
+	}
+
+	if err := os.RemoveAll(nns.symlink.Name()); err != nil {
+		return fmt.Errorf("failed to remove net ns symlink: %v", err)
+	}
+
+	return nil
+}
+
+func hostNetNsPath() (string, error) {
+	netNS, err := ns.GetCurrentNS()
+	if err != nil {
+		return "", err
+	}
+
+	defer netNS.Close()
+	return netNS.Path(), nil
+}

lib/sandbox/sandbox_unsupported.go

@@ -0,0 +1,49 @@
+// +build !linux
+
+package sandbox
+
+import (
+	"os"
+)
+
+func isNSorErr(nspath string) error {
+	return nil // TODO(vbatts) ... really not sure ...
+}
+
+func newNetNs() (*NetNs, error) {
+	return &NetNs{}, nil
+}
+
+func getNetNs(path string) (*NetNs, error) {
+	return &NetNs{}, nil
+}
+
+// NetNs handles data pertaining a network namespace
+// for non-linux this is a noop
+type NetNs struct {
+	symlink *os.File
+}
+
+func (nns *NetNs) Path() string {
+	return ""
+}
+
+func (nns *NetNs) symlinkCreate(name string) error {
+	return nil
+}
+
+func (nns *NetNs) symlinkRemove() error {
+	return nil
+}
+
+func (nns *NetNs) Close() error {
+	return nil
+}
+
+func (nns *NetNs) Remove() error {
+	return nil
+}
+
+func hostNetNsPath() (string, error) {
+	return "", nil // TODO(vbatts) maybe this should have a platform error?
+}

lib/stats.go

@@ -1,22 +1,15 @@
 package lib
 
 import (
-	"path/filepath"
-	"syscall"
-	"time"
-
-	"strings"
-
 	"github.com/kubernetes-incubator/cri-o/oci"
-	"github.com/opencontainers/runc/libcontainer"
 )
 
 // ContainerStats contains the statistics information for a running container
 type ContainerStats struct {
 	Container   string
 	CPU         float64
-	cpuNano     uint64
-	systemNano  uint64
+	CPUNano     uint64
+	SystemNano  int64
 	MemUsage    uint64
 	MemLimit    uint64
 	MemPerc     float64
@@ -29,83 +22,5 @@ type ContainerStats struct {
 
 // GetContainerStats gets the running stats for a given container
 func (c *ContainerServer) GetContainerStats(ctr *oci.Container, previousStats *ContainerStats) (*ContainerStats, error) {
-	previousCPU := previousStats.cpuNano
-	previousSystem := previousStats.systemNano
-	libcontainerStats, err := c.LibcontainerStats(ctr)
-	if err != nil {
-		return nil, err
-	}
-	cgroupStats := libcontainerStats.CgroupStats
-	stats := new(ContainerStats)
-	stats.Container = ctr.ID()
-	stats.CPU = calculateCPUPercent(libcontainerStats, previousCPU, previousSystem)
-	stats.MemUsage = cgroupStats.MemoryStats.Usage.Usage
-	stats.MemLimit = getMemLimit(cgroupStats.MemoryStats.Usage.Limit)
-	stats.MemPerc = float64(stats.MemUsage) / float64(stats.MemLimit)
-	stats.PIDs = cgroupStats.PidsStats.Current
-	stats.BlockInput, stats.BlockOutput = calculateBlockIO(libcontainerStats)
-	stats.NetInput, stats.NetOutput = getContainerNetIO(libcontainerStats)
-
-	return stats, nil
-}
-
-func loadFactory(root string) (libcontainer.Factory, error) {
-	abs, err := filepath.Abs(root)
-	if err != nil {
-		return nil, err
-	}
-	cgroupManager := libcontainer.Cgroupfs
-	return libcontainer.New(abs, cgroupManager, libcontainer.CriuPath(""))
-}
-
-// getMemory limit returns the memory limit for a given cgroup
-// If the configured memory limit is larger than the total memory on the sys, the
-// physical system memory size is returned
-func getMemLimit(cgroupLimit uint64) uint64 {
-	si := &syscall.Sysinfo_t{}
-	err := syscall.Sysinfo(si)
-	if err != nil {
-		return cgroupLimit
-	}
-
-	physicalLimit := uint64(si.Totalram)
-	if cgroupLimit > physicalLimit {
-		return physicalLimit
-	}
-	return cgroupLimit
-}
-
-// Returns the total number of bytes transmitted and received for the given container stats
-func getContainerNetIO(stats *libcontainer.Stats) (received uint64, transmitted uint64) {
-	for _, iface := range stats.Interfaces {
-		received += iface.RxBytes
-		transmitted += iface.TxBytes
-	}
-	return
-}
-
-func calculateCPUPercent(stats *libcontainer.Stats, previousCPU, previousSystem uint64) float64 {
-	var (
-		cpuPercent  = 0.0
-		cpuDelta    = float64(stats.CgroupStats.CpuStats.CpuUsage.TotalUsage - previousCPU)
-		systemDelta = float64(uint64(time.Now().UnixNano()) - previousSystem)
-	)
-	if systemDelta > 0.0 && cpuDelta > 0.0 {
-		// gets a ratio of container cpu usage total, multiplies it by the number of cores (4 cores running
-		// at 100% utilization should be 400% utilization), and multiplies that by 100 to get a percentage
-		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.CgroupStats.CpuStats.CpuUsage.PercpuUsage)) * 100
-	}
-	return cpuPercent
-}
-
-func calculateBlockIO(stats *libcontainer.Stats) (read uint64, write uint64) {
-	for _, blkIOEntry := range stats.CgroupStats.BlkioStats.IoServiceBytesRecursive {
-		switch strings.ToLower(blkIOEntry.Op) {
-		case "read":
-			read += blkIOEntry.Value
-		case "write":
-			write += blkIOEntry.Value
-		}
-	}
-	return
+	return c.getContainerStats(ctr, previousStats)
 }

lib/stats_linux.go

@@ -0,0 +1,61 @@
+package lib
+
+import (
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/opencontainers/runc/libcontainer"
+)
+
+// Returns the total number of bytes transmitted and received for the given container stats
+func getContainerNetIO(stats *libcontainer.Stats) (received uint64, transmitted uint64) {
+	for _, iface := range stats.Interfaces {
+		received += iface.RxBytes
+		transmitted += iface.TxBytes
+	}
+	return
+}
+
+func calculateCPUPercent(stats *libcontainer.Stats, previousCPU uint64, previousSystem int64) float64 {
+	var (
+		cpuPercent  = 0.0
+		cpuDelta    = float64(stats.CgroupStats.CpuStats.CpuUsage.TotalUsage - previousCPU)
+		systemDelta = float64(uint64(time.Now().UnixNano()) - uint64(previousSystem))
+	)
+	if systemDelta > 0.0 && cpuDelta > 0.0 {
+		// gets a ratio of container cpu usage total, multiplies it by the number of cores (4 cores running
+		// at 100% utilization should be 400% utilization), and multiplies that by 100 to get a percentage
+		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.CgroupStats.CpuStats.CpuUsage.PercpuUsage)) * 100
+	}
+	return cpuPercent
+}
+
+func calculateBlockIO(stats *libcontainer.Stats) (read uint64, write uint64) {
+	for _, blkIOEntry := range stats.CgroupStats.BlkioStats.IoServiceBytesRecursive {
+		switch strings.ToLower(blkIOEntry.Op) {
+		case "read":
+			read += blkIOEntry.Value
+		case "write":
+			write += blkIOEntry.Value
+		}
+	}
+	return
+}
+
+// getMemory limit returns the memory limit for a given cgroup
+// If the configured memory limit is larger than the total memory on the sys, the
+// physical system memory size is returned
+func getMemLimit(cgroupLimit uint64) uint64 {
+	si := &syscall.Sysinfo_t{}
+	err := syscall.Sysinfo(si)
+	if err != nil {
+		return cgroupLimit
+	}
+
+	physicalLimit := uint64(si.Totalram)
+	if cgroupLimit > physicalLimit {
+		return physicalLimit
+	}
+	return cgroupLimit
+}

lib/stop.go

@@ -24,6 +24,9 @@ func (c *ContainerServer) ContainerStop(ctx context.Context, container string, t
 			if err := c.runtime.StopContainer(ctx, ctr, timeout); err != nil {
 				return "", errors.Wrapf(err, "failed to stop container %s", ctrID)
 			}
+			if err := c.runtime.WaitContainerStateStopped(ctx, ctr, timeout); err != nil {
+				return "", errors.Wrapf(err, "failed to get container 'stopped' status %s", ctrID)
+			}
 			if err := c.storageRuntimeServer.StopContainer(ctrID); err != nil {
 				return "", errors.Wrapf(err, "failed to unmount container %s", ctrID)
 			}

lib/testdata/config.toml

@@ -16,7 +16,7 @@
     cgroup_manager = "cgroupfs"
     hooks_dir_path = "/usr/share/containers/oci/hooks.d"
     pids_limit = 2048
-    container_exits_dir = "/var/run/kpod/exits"
+    container_exits_dir = "/var/run/podman/exits"
   [crio.image]
     default_transport = "docker://"
     pause_image = "kubernetes/pause"

oci/container.go

@@ -9,11 +9,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/docker/docker/pkg/signal"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"k8s.io/apimachinery/pkg/fields"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 const (
@@ -30,7 +29,7 @@ type Container struct {
 	crioAnnotations fields.Set
 	image           string
 	sandbox         string
-	netns           ns.NetNS
+	netns           string
 	terminal        bool
 	stdin           bool
 	stdinOnce       bool
@@ -71,7 +70,7 @@ type ContainerState struct {
 }
 
 // NewContainer creates a container object.
-func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) {
+func NewContainer(id string, name string, bundlePath string, logPath string, netns string, labels map[string]string, crioAnnotations map[string]string, annotations map[string]string, image string, imageName string, imageRef string, metadata *pb.ContainerMetadata, sandbox string, terminal bool, stdin bool, stdinOnce bool, privileged bool, trusted bool, dir string, created time.Time, stopSignal string) (*Container, error) {
 	state := &ContainerState{}
 	state.Created = created
 	c := &Container{
@@ -223,11 +222,11 @@ func (c *Container) NetNsPath() (string, error) {
 		return "", fmt.Errorf("container state is not populated")
 	}
 
-	if c.netns == nil {
+	if c.netns == "" {
 		return fmt.Sprintf("/proc/%d/ns/net", c.state.Pid), nil
 	}
 
-	return c.netns.Path(), nil
+	return c.netns, nil
 }
 
 // Metadata returns the metadata of the container.

oci/finished.go

@@ -1,4 +1,4 @@
-// +build !arm,!386
+// +build linux,!arm,!386
 
 package oci
 

oci/finished_32.go

@@ -1,4 +1,4 @@
-// +build arm 386
+// +build linux,arm linux,386
 
 package oci
 

oci/finished_unsupported.go

@@ -0,0 +1,23 @@
+// +build !linux
+
+package oci
+
+import (
+	"os"
+	"time"
+)
+
+func getFinishedTime(fi os.FileInfo) time.Time {
+	// Windows would be like
+	//st := fi.Sys().(*syscall.Win32FileAttributeDatao)
+	//st.CreationTime.Nanoseconds()
+
+	// Darwin and Freebsd would be like
+	//st := fi.Sys().(*syscall.Stat_t)
+	//st.Ctimespec.Nsec
+
+	// openbsd would be like
+	//st := fi.Sys().(*syscall.Stat_t)
+	//st.Ctim.Nsec
+	return fi.ModTime()
+}

oci/oci.go

@@ -13,12 +13,10 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/containerd/cgroups"
 	"github.com/kubernetes-incubator/cri-o/utils"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	"golang.org/x/sys/unix"
 	kwait "k8s.io/apimachinery/pkg/util/wait"
 )
 
@@ -189,6 +187,9 @@ func (r *Runtime) CreateContainer(c *Container, cgroupParent string) (err error)
 	if c.terminal {
 		args = append(args, "-t")
 	} else if c.stdin {
+		if !c.stdinOnce {
+			args = append(args, "--leave-stdin-open")
+		}
 		args = append(args, "-i")
 	}
 	logrus.WithFields(logrus.Fields{
@@ -197,9 +198,7 @@ func (r *Runtime) CreateContainer(c *Container, cgroupParent string) (err error)
 
 	cmd := exec.Command(r.conmonPath, args...)
 	cmd.Dir = c.bundlePath
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
-	}
+	cmd.SysProcAttr = sysProcAttrPlatform()
 	cmd.Stdin = os.Stdin
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -228,19 +227,8 @@ func (r *Runtime) CreateContainer(c *Container, cgroupParent string) (err error)
 			logrus.Warnf("Failed to add conmon to systemd sandbox cgroup: %v", err)
 		}
 	} else {
-		control, err := cgroups.New(cgroups.V1, cgroups.StaticPath(filepath.Join(cgroupParent, "/crio-conmon-"+c.id)), &rspec.LinuxResources{})
-		if err != nil {
-			logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
-		} else {
-			// Here we should defer a crio-connmon- cgroup hierarchy deletion, but it will
-			// always fail as conmon's pid is still there.
-			// Fortunately, kubelet takes care of deleting this for us, so the leak will
-			// only happens in corner case where one does a manual deletion of the container
-			// through e.g. runc. This should be handled by implementing a conmon monitoring
-			// routine that does the cgroup cleanup once conmon is terminated.
-			if err := control.Add(cgroups.Process{Pid: cmd.Process.Pid}); err != nil {
-				logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
-			}
+		if err := createContainerPlatform(c, cgroupParent, cmd.Process.Pid); err != nil {
+			logrus.Warnf("%s", err)
 		}
 	}
 
@@ -438,7 +426,7 @@ func (r *Runtime) ExecSync(c *Container, command []string, timeout int64) (resp
 	args = append(args, "-l", logPath)
 	args = append(args, "--socket-dir-path", ContainerAttachSocketDir)
 
-	processFile, err := PrepareProcessExec(c, command, false)
+	processFile, err := PrepareProcessExec(c, command, c.terminal)
 	if err != nil {
 		return nil, ExecSyncError{
 			ExitCode: -1,
@@ -475,7 +463,7 @@ func (r *Runtime) ExecSync(c *Container, command []string, timeout int64) (resp
 	err = cmd.Wait()
 	if err != nil {
 		if exitErr, ok := err.(*exec.ExitError); ok {
-			if status, ok := exitErr.Sys().(unix.WaitStatus); ok {
+			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
 				return nil, ExecSyncError{
 					Stdout:   stdoutBuf,
 					Stderr:   stderrBuf,
@@ -560,12 +548,12 @@ func (r *Runtime) UpdateContainer(c *Container, res *rspec.LinuxResources) error
 
 func waitContainerStop(ctx context.Context, c *Container, timeout time.Duration) error {
 	done := make(chan struct{})
-	// we could potentially re-use "done" channel to exit the loop on timeout
-	// but we use another channel "chControl" so that we won't never incur in the
-	// case the "done" channel is closed in the "default" select case and we also
-	// reach the timeout in the select below. If that happens we could raise
-	// a panic closing a closed channel so better be safe and use another new
-	// channel just to control the loop.
+	// we could potentially re-use "done" channel to exit the loop on timeout,
+	// but we use another channel "chControl" so that we never panic
+	// attempting to close an already-closed "done" channel.  The panic
+	// would occur in the "default" select case below if we'd closed the
+	// "done" channel (instead of the "chControl" channel) in the timeout
+	// select case.
 	chControl := make(chan struct{})
 	go func() {
 		for {
@@ -574,8 +562,13 @@ func waitContainerStop(ctx context.Context, c *Container, timeout time.Duration)
 				return
 			default:
 				// Check if the process is still around
-				err := unix.Kill(c.state.Pid, 0)
-				if err == unix.ESRCH {
+				proc, err := os.FindProcess(c.state.Pid)
+				if err != nil {
+					close(done)
+					return
+				}
+				err = proc.Signal(syscall.Signal(0))
+				if err == syscall.ESRCH {
 					close(done)
 					return
 				}
@@ -591,8 +584,12 @@ func waitContainerStop(ctx context.Context, c *Container, timeout time.Duration)
 		return ctx.Err()
 	case <-time.After(timeout):
 		close(chControl)
-		err := unix.Kill(c.state.Pid, unix.SIGKILL)
-		if err != nil && err != unix.ESRCH {
+		proc, err := os.FindProcess(c.state.Pid)
+		if err != nil {
+			return fmt.Errorf("failed to find pid %d: %v", c.state.Pid, err)
+		}
+		err = proc.Signal(syscall.SIGKILL)
+		if err != nil && err != syscall.ESRCH {
 			return fmt.Errorf("failed to kill process: %v", err)
 		}
 	}
@@ -601,14 +598,68 @@ func waitContainerStop(ctx context.Context, c *Container, timeout time.Duration)
 	return nil
 }
 
+// WaitContainerStateStopped runs a loop polling UpdateStatus(), seeking for
+// the container status to be updated to 'stopped'. Either it gets the expected
+// status and returns nil, or it reaches the timeout and returns an error.
+func (r *Runtime) WaitContainerStateStopped(ctx context.Context, c *Container, timeout int64) (err error) {
+	// No need to go further and spawn the go routine if the container
+	// is already in the expected status.
+	if r.ContainerStatus(c).Status == ContainerStateStopped {
+		return nil
+	}
+
+	done := make(chan error)
+	chControl := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-chControl:
+				return
+			default:
+				// Check if the container is stopped
+				if err := r.UpdateStatus(c); err != nil {
+					done <- err
+					close(done)
+					return
+				}
+				if r.ContainerStatus(c).Status == ContainerStateStopped {
+					close(done)
+					return
+				}
+				time.Sleep(100 * time.Millisecond)
+			}
+		}
+	}()
+	select {
+	case err = <-done:
+		break
+	case <-ctx.Done():
+		close(chControl)
+		return ctx.Err()
+	case <-time.After(time.Duration(timeout) * time.Second):
+		close(chControl)
+		return fmt.Errorf("failed to get container stopped status: %ds timeout reached", timeout)
+	}
+
+	if err != nil {
+		return fmt.Errorf("failed to get container stopped status: %v", err)
+	}
+
+	return nil
+}
+
 // StopContainer stops a container. Timeout is given in seconds.
 func (r *Runtime) StopContainer(ctx context.Context, c *Container, timeout int64) error {
 	c.opLock.Lock()
 	defer c.opLock.Unlock()
 
 	// Check if the process is around before sending a signal
-	err := unix.Kill(c.state.Pid, 0)
-	if err == unix.ESRCH {
+	proc, err := os.FindProcess(c.state.Pid)
+	if err != nil {
+		return nil
+	}
+	err = proc.Signal(syscall.Signal(0))
+	if err == syscall.ESRCH {
 		c.state.Finished = time.Now()
 		return nil
 	}
@@ -652,6 +703,7 @@ func (r *Runtime) SetStartFailed(c *Container, err error) {
 func (r *Runtime) UpdateStatus(c *Container) error {
 	c.opLock.Lock()
 	defer c.opLock.Unlock()
+
 	out, err := exec.Command(r.Path(c), "state", c.id).Output()
 	if err != nil {
 		// there are many code paths that could lead to have a bad state in the
@@ -688,7 +740,7 @@ func (r *Runtime) UpdateStatus(c *Container) error {
 				return true, nil
 			})
 		if err != nil {
-			logrus.Warnf("failed to find container exit file: %v", err)
+			logrus.Warnf("failed to find container exit file for %v: %v", c.id, err)
 			c.state.ExitCode = -1
 		} else {
 			c.state.Finished = getFinishedTime(fi)
@@ -719,15 +771,6 @@ func (r *Runtime) ContainerStatus(c *Container) *ContainerState {
 	return c.state
 }
 
-// newPipe creates a unix socket pair for communication
-func newPipe() (parent *os.File, child *os.File, err error) {
-	fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
-	if err != nil {
-		return nil, nil, err
-	}
-	return os.NewFile(uintptr(fds[1]), "parent"), os.NewFile(uintptr(fds[0]), "child"), nil
-}
-
 // RuntimeReady checks if the runtime is up and ready to accept
 // basic containers e.g. container only needs host network.
 func (r *Runtime) RuntimeReady() (bool, error) {

oci/oci_linux.go

@@ -0,0 +1,47 @@
+// +build linux
+
+package oci
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/containerd/cgroups"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"golang.org/x/sys/unix"
+)
+
+func createContainerPlatform(c *Container, cgroupParent string, pid int) error {
+	control, err := cgroups.New(cgroups.V1, cgroups.StaticPath(filepath.Join(cgroupParent, "/crio-conmon-"+c.id)), &rspec.LinuxResources{})
+	if err != nil {
+		return fmt.Errorf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
+	} else {
+		// Here we should defer a crio-connmon- cgroup hierarchy deletion, but it will
+		// always fail as conmon's pid is still there.
+		// Fortunately, kubelet takes care of deleting this for us, so the leak will
+		// only happens in corner case where one does a manual deletion of the container
+		// through e.g. runc. This should be handled by implementing a conmon monitoring
+		// routine that does the cgroup cleanup once conmon is terminated.
+		if err := control.Add(cgroups.Process{Pid: pid}); err != nil {
+			fmt.Errorf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
+		}
+	}
+	return nil
+}
+
+func sysProcAttrPlatform() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+}
+
+// newPipe creates a unix socket pair for communication
+func newPipe() (parent *os.File, child *os.File, err error) {
+	fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	return os.NewFile(uintptr(fds[1]), "parent"), os.NewFile(uintptr(fds[0]), "child"), nil
+}

oci/oci_unsupported.go

@@ -0,0 +1,20 @@
+// +build !linux
+
+package oci
+
+import (
+	"os"
+	"syscall"
+)
+
+func createContainerPlatform(c *Container, cgroupParent string, pid int) error {
+	return nil
+}
+
+func sysProcAttrPlatform() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{}
+}
+
+func newPipe() (parent *os.File, child *os.File, err error) {
+	return os.Pipe()
+}

pause/Makefile

@@ -6,7 +6,6 @@ override CFLAGS += -std=c99 -Os -Wall -Wextra -static
 
 pause: $(obj)
 	$(CC) -o ../bin/$@ $^ $(CFLAGS) $(LIBS)
-	strip ../bin/$@
 
 .PHONY: clean
 clean:

pkg/annotations/annotations.go

@@ -19,9 +19,15 @@ const (
 	// HostName is the container host name annotation
 	HostName = "io.kubernetes.cri-o.HostName"
 
+	// CgroupParent is the sandbox cgroup parent
+	CgroupParent = "io.kubernetes.cri-o.CgroupParent"
+
 	// IP is the container ipv4 or ipv6 address
 	IP = "io.kubernetes.cri-o.IP"
 
+	// NamespaceOptions store the options for namespaces
+	NamespaceOptions = "io.kubernetes.cri-o.NamespaceOptions"
+
 	// SeccompProfilePath is the node seccomp profile path
 	SeccompProfilePath = "io.kubernetes.cri-o.SeccompProfilePath"
 
@@ -37,6 +43,9 @@ const (
 	// KubeName is the kubernetes name annotation
 	KubeName = "io.kubernetes.cri-o.KubeName"
 
+	// PortMappings holds the port mappings for the sandbox
+	PortMappings = "io.kubernetes.cri-o.PortMappings"
+
 	// Labels are the kubernetes labels annotation
 	Labels = "io.kubernetes.cri-o.Labels"
 
@@ -49,6 +58,9 @@ const (
 	// Name is the pod name annotation
 	Name = "io.kubernetes.cri-o.Name"
 
+	// Namespace is the pod namespace annotation
+	Namespace = "io.kubernetes.cri-o.Namespace"
+
 	// PrivilegedRuntime is the annotation for the privileged runtime path
 	PrivilegedRuntime = "io.kubernetes.cri-o.PrivilegedRuntime"
 

pkg/storage/image.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"path"
 	"strings"
+	"sync"
 
 	"github.com/containers/image/copy"
 	"github.com/containers/image/docker/reference"
@@ -39,6 +40,7 @@ type ImageResult struct {
 	Size         *uint64
 	Digest       digest.Digest
 	ConfigDigest digest.Digest
+	User         string
 }
 
 type indexInfo struct {
@@ -46,12 +48,23 @@ type indexInfo struct {
 	secure bool
 }
 
+// A set of information that we prefer to cache about images, so that we can
+// avoid having to reread them every time we need to return information about
+// images.
+type imageCacheItem struct {
+	user         string
+	size         *uint64
+	configDigest digest.Digest
+}
+
 type imageService struct {
 	store                 storage.Store
 	defaultTransport      string
 	insecureRegistryCIDRs []*net.IPNet
 	indexConfigs          map[string]*indexInfo
 	registries            []string
+	imageCache            map[string]imageCacheItem
+	imageCacheLock        sync.Mutex
 }
 
 // sizer knows its size.
@@ -171,15 +184,38 @@ func (svc *imageService) ListImages(systemContext *types.SystemContext, filter s
 			return nil, err
 		}
 		if image, err := istorage.Transport.GetStoreImage(svc.store, ref); err == nil {
-			img, err := ref.NewImageSource(systemContext)
-			if err != nil {
-				return nil, err
-			}
-			size := imageSize(img)
-			configDigest, err := imageConfigDigest(img, nil)
-			img.Close()
-			if err != nil {
-				return nil, err
+			var user string
+			var size *uint64
+			var configDigest digest.Digest
+			if cacheItem, ok := svc.imageCache[image.ID]; ok {
+				user, size, configDigest = cacheItem.user, cacheItem.size, cacheItem.configDigest
+			} else {
+				img, err := ref.NewImageSource(systemContext)
+				if err != nil {
+					return nil, err
+				}
+				size = imageSize(img)
+				configDigest, err = imageConfigDigest(img, nil)
+				img.Close()
+				if err != nil {
+					return nil, err
+				}
+				imageFull, err := ref.NewImage(systemContext)
+				if err != nil {
+					return nil, err
+				}
+				defer imageFull.Close()
+				imageConfig, err := imageFull.OCIConfig()
+				if err != nil {
+					return nil, err
+				}
+				user = imageConfig.Config.User
+				cacheItem := imageCacheItem{
+					user:         user,
+					size:         size,
+					configDigest: configDigest,
+				}
+				svc.imageCache[image.ID] = cacheItem
 			}
 			name, tags, digests := sortNamesByType(image.Names)
 			imageDigest, repoDigests := svc.makeRepoDigests(digests, tags, image.ID)
@@ -191,6 +227,7 @@ func (svc *imageService) ListImages(systemContext *types.SystemContext, filter s
 				Size:         size,
 				Digest:       imageDigest,
 				ConfigDigest: configDigest,
+				User:         user,
 			})
 		}
 	} else {
@@ -198,20 +235,72 @@ func (svc *imageService) ListImages(systemContext *types.SystemContext, filter s
 		if err != nil {
 			return nil, err
 		}
+		visited := make(map[string]struct{})
+		defer func() {
+			// We built a map using IDs of images that we looked
+			// at, so remove any items from the cache that don't
+			// correspond to any of those IDs.
+			removedIDs := make([]string, 0, len(svc.imageCache))
+			for imageID := range svc.imageCache {
+				if _, keep := visited[imageID]; !keep {
+					// We have cached data for an image
+					// with this ID, but it's not in the
+					// list of images now, so the image has
+					// been removed.
+					removedIDs = append(removedIDs, imageID)
+				}
+			}
+			// Handle the removals.
+			svc.imageCacheLock.Lock()
+			for _, removedID := range removedIDs {
+				delete(svc.imageCache, removedID)
+			}
+			svc.imageCacheLock.Unlock()
+		}()
 		for _, image := range images {
-			ref, err := istorage.Transport.ParseStoreReference(svc.store, "@"+image.ID)
-			if err != nil {
-				return nil, err
-			}
-			img, err := ref.NewImageSource(systemContext)
-			if err != nil {
-				return nil, err
-			}
-			size := imageSize(img)
-			configDigest, err := imageConfigDigest(img, nil)
-			img.Close()
-			if err != nil {
-				return nil, err
+			visited[image.ID] = struct{}{}
+			var user string
+			var size *uint64
+			var configDigest digest.Digest
+			svc.imageCacheLock.Lock()
+			cacheItem, ok := svc.imageCache[image.ID]
+			svc.imageCacheLock.Unlock()
+			if ok {
+				user, size, configDigest = cacheItem.user, cacheItem.size, cacheItem.configDigest
+			} else {
+				ref, err := istorage.Transport.ParseStoreReference(svc.store, "@"+image.ID)
+				if err != nil {
+					return nil, err
+				}
+				img, err := ref.NewImageSource(systemContext)
+				if err != nil {
+					return nil, err
+				}
+				size = imageSize(img)
+				configDigest, err = imageConfigDigest(img, nil)
+				img.Close()
+				if err != nil {
+					return nil, err
+				}
+				imageFull, err := ref.NewImage(systemContext)
+				if err != nil {
+					return nil, err
+				}
+				defer imageFull.Close()
+
+				imageConfig, err := imageFull.OCIConfig()
+				if err != nil {
+					return nil, err
+				}
+				user = imageConfig.Config.User
+				cacheItem := imageCacheItem{
+					user:         user,
+					size:         size,
+					configDigest: configDigest,
+				}
+				svc.imageCacheLock.Lock()
+				svc.imageCache[image.ID] = cacheItem
+				svc.imageCacheLock.Unlock()
 			}
 			name, tags, digests := sortNamesByType(image.Names)
 			imageDigest, repoDigests := svc.makeRepoDigests(digests, tags, image.ID)
@@ -223,6 +312,7 @@ func (svc *imageService) ListImages(systemContext *types.SystemContext, filter s
 				Size:         size,
 				Digest:       imageDigest,
 				ConfigDigest: configDigest,
+				User:         user,
 			})
 		}
 	}
@@ -230,22 +320,24 @@ func (svc *imageService) ListImages(systemContext *types.SystemContext, filter s
 }
 
 func (svc *imageService) ImageStatus(systemContext *types.SystemContext, nameOrID string) (*ImageResult, error) {
-	ref, err := alltransports.ParseImageName(nameOrID)
+	ref, err := svc.getRef(nameOrID)
 	if err != nil {
-		ref2, err2 := istorage.Transport.ParseStoreReference(svc.store, "@"+nameOrID)
-		if err2 != nil {
-			ref3, err3 := istorage.Transport.ParseStoreReference(svc.store, nameOrID)
-			if err3 != nil {
-				return nil, err
-			}
-			ref2 = ref3
-		}
-		ref = ref2
+		return nil, err
 	}
 	image, err := istorage.Transport.GetStoreImage(svc.store, ref)
 	if err != nil {
 		return nil, err
 	}
+	imageFull, err := ref.NewImage(systemContext)
+	if err != nil {
+		return nil, err
+	}
+	defer imageFull.Close()
+
+	imageConfig, err := imageFull.OCIConfig()
+	if err != nil {
+		return nil, err
+	}
 
 	img, err := ref.NewImageSource(systemContext)
 	if err != nil {
@@ -268,6 +360,7 @@ func (svc *imageService) ImageStatus(systemContext *types.SystemContext, nameOrI
 		Size:         size,
 		Digest:       imageDigest,
 		ConfigDigest: configDigest,
+		User:         imageConfig.Config.User,
 	}
 
 	return &result, nil
@@ -399,19 +492,10 @@ func (svc *imageService) PullImage(systemContext *types.SystemContext, imageName
 }
 
 func (svc *imageService) UntagImage(systemContext *types.SystemContext, nameOrID string) error {
-	ref, err := alltransports.ParseImageName(nameOrID)
+	ref, err := svc.getRef(nameOrID)
 	if err != nil {
-		ref2, err2 := istorage.Transport.ParseStoreReference(svc.store, "@"+nameOrID)
-		if err2 != nil {
-			ref3, err3 := istorage.Transport.ParseStoreReference(svc.store, nameOrID)
-			if err3 != nil {
-				return err
-			}
-			ref2 = ref3
-		}
-		ref = ref2
+		return err
 	}
-
 	img, err := istorage.Transport.GetStoreImage(svc.store, ref)
 	if err != nil {
 		return err
@@ -450,17 +534,9 @@ func (svc *imageService) UntagImage(systemContext *types.SystemContext, nameOrID
 }
 
 func (svc *imageService) RemoveImage(systemContext *types.SystemContext, nameOrID string) error {
-	ref, err := alltransports.ParseImageName(nameOrID)
+	ref, err := svc.getRef(nameOrID)
 	if err != nil {
-		ref2, err2 := istorage.Transport.ParseStoreReference(svc.store, "@"+nameOrID)
-		if err2 != nil {
-			ref3, err3 := istorage.Transport.ParseStoreReference(svc.store, nameOrID)
-			if err3 != nil {
-				return err
-			}
-			ref2 = ref3
-		}
-		ref = ref2
+		return err
 	}
 	return ref.DeleteImage(systemContext)
 }
@@ -515,6 +591,8 @@ func splitDockerDomain(name string) (domain, remainder string) {
 	return
 }
 
+// ResolveNames resolves an image name into a storage image ID or a fully-qualified image name (domain/repo/image:tag).
+// Will only return an empty slice if err != nil.
 func (svc *imageService) ResolveNames(imageName string) ([]string, error) {
 	// _Maybe_ it's a truncated image ID.  Don't prepend a registry name, then.
 	if len(imageName) >= minimumTruncatedIDLength && svc.store != nil {
@@ -585,6 +663,7 @@ func GetImageService(store storage.Store, defaultTransport string, insecureRegis
 		indexConfigs:          make(map[string]*indexInfo, 0),
 		insecureRegistryCIDRs: make([]*net.IPNet, 0),
 		registries:            cleanRegistries,
+		imageCache:            make(map[string]imageCacheItem),
 	}
 
 	insecureRegistries = append(insecureRegistries, "127.0.0.0/8")

releases/README.md

@@ -0,0 +1,8 @@
+## `releases` ##
+
+```
+make install-tools
+git tag vX.Y.Z <commit>
+touch releases/vX.Y.Z.toml
+make release-note release=releases/vX.Y.Z.toml
+```

releases/v1.0.9.toml

@@ -0,0 +1,13 @@
+# The commit/tag of the release
+commit = "v1.0.9"
+project_name = "CRI-O"
+github_repo = "kubernetes-incubator/cri-o"
+previous = "v1.0.8"
+pre_release = false
+
+preface = """\
+"""
+
+[notes]
+
+[breaking]

releases/v1.8.5.toml

@@ -0,0 +1,13 @@
+# The commit/tag of the release
+commit = "v1.8.5"
+project_name = "CRI-O"
+github_repo = "kubernetes-incubator/cri-o"
+previous = "v1.8.4"
+pre_release = false
+
+preface = """\
+"""
+
+[notes]
+
+[breaking]

releases/v1.9.1.toml

@@ -0,0 +1,13 @@
+# The commit/tag of the release
+commit = "v1.9.1"
+project_name = "CRI-O"
+github_repo = "kubernetes-incubator/cri-o"
+previous = "v1.9.0"
+pre_release = false
+
+preface = """\
+"""
+
+[notes]
+
+[breaking]

releases/v1.9.2.toml

@@ -0,0 +1,13 @@
+# The commit/tag of the release
+commit = "v1.9.2"
+project_name = "CRI-O"
+github_repo = "kubernetes-incubator/cri-o"
+previous = "v1.9.1"
+pre_release = false
+
+preface = """\
+"""
+
+[notes]
+
+[breaking]

releases/v1.9.3.toml

@@ -0,0 +1,13 @@
+# The commit/tag of the release
+commit = "v1.9.3"
+project_name = "CRI-O"
+github_repo = "kubernetes-incubator/cri-o"
+previous = "v1.9.2"
+pre_release = false
+
+preface = """\
+"""
+
+[notes]
+
+[breaking]

releases/v1.9.5.toml

@@ -0,0 +1,13 @@
+# The commit/tag of the release
+commit = "v1.9.5"
+project_name = "CRI-O"
+github_repo = "kubernetes-incubator/cri-o"
+previous = "v1.9.3"
+pre_release = false
+
+preface = """\
+"""
+
+[notes]
+
+[breaking]

releases/v1.9.6.toml

@@ -0,0 +1,13 @@
+# The commit/tag of the release
+commit = "v1.9.6"
+project_name = "CRI-O"
+github_repo = "kubernetes-incubator/cri-o"
+previous = "v1.9.5"
+pre_release = false
+
+preface = """\
+"""
+
+[notes]
+
+[breaking]

server/config_test.go

@@ -10,12 +10,6 @@ import (
 
 const fixturePath = "fixtures/crio.conf"
 
-func must(t *testing.T, err error) {
-	if err != nil {
-		t.Error(err)
-	}
-}
-
 func assertAllFieldsEquality(t *testing.T, c Config) {
 	testCases := []struct {
 		fieldValue, expected interface{}

server/container_attach.go

@@ -6,15 +6,15 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"syscall"
 	"time"
 
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/kubernetes-incubator/cri-o/utils"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	"golang.org/x/sys/unix"
 	"k8s.io/client-go/tools/remotecommand"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 )
 
@@ -60,10 +60,11 @@ func (ss streamService) Attach(containerID string, inputStream io.Reader, output
 	}
 
 	controlPath := filepath.Join(c.BundlePath(), "ctl")
-	controlFile, err := os.OpenFile(controlPath, unix.O_WRONLY, 0)
+	controlFile, err := os.OpenFile(controlPath, syscall.O_WRONLY, 0)
 	if err != nil {
 		return fmt.Errorf("failed to open container ctl file: %v", err)
 	}
+	defer controlFile.Close()
 
 	kubecontainer.HandleResizing(resize, func(size remotecommand.TerminalSize) {
 		logrus.Infof("Got a resize event: %+v", size)

server/container_create.go

@@ -12,6 +12,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	dockermounts "github.com/docker/docker/pkg/mount"
@@ -25,16 +26,13 @@ import (
 	"github.com/kubernetes-incubator/cri-o/server/apparmor"
 	"github.com/kubernetes-incubator/cri-o/server/seccomp"
 	"github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/opencontainers/runc/libcontainer/cgroups"
-	"github.com/opencontainers/runc/libcontainer/devices"
 	"github.com/opencontainers/runc/libcontainer/user"
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	"golang.org/x/sys/unix"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 const (
@@ -136,7 +134,7 @@ func addOCIBindMounts(mountLabel string, containerConfig *pb.ContainerConfig, sp
 
 		if mount.SelinuxRelabel {
 			// Need a way in kubernetes to determine if the volume is shared or private
-			if err := label.Relabel(src, mountLabel, true); err != nil && err != unix.ENOTSUP {
+			if err := label.Relabel(src, mountLabel, true); err != nil && err != syscall.ENOTSUP {
 				return nil, nil, fmt.Errorf("relabel failed %s: %v", src, err)
 			}
 		}
@@ -245,7 +243,7 @@ func addImageVolumes(rootfs string, s *Server, containerInfo *storage.ContainerI
 			}
 			// Label the source with the sandbox selinux mount label
 			if mountLabel != "" {
-				if err1 := label.Relabel(src, mountLabel, true); err1 != nil && err1 != unix.ENOTSUP {
+				if err1 := label.Relabel(src, mountLabel, true); err1 != nil && err1 != syscall.ENOTSUP {
 					return nil, fmt.Errorf("relabel failed %s: %v", src, err1)
 				}
 			}
@@ -280,99 +278,7 @@ func resolveSymbolicLink(path string) (string, error) {
 }
 
 func addDevices(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
-	sp := specgen.Spec()
-	if containerConfig.GetLinux().GetSecurityContext().GetPrivileged() {
-		hostDevices, err := devices.HostDevices()
-		if err != nil {
-			return err
-		}
-		for _, hostDevice := range hostDevices {
-			rd := rspec.LinuxDevice{
-				Path:  hostDevice.Path,
-				Type:  string(hostDevice.Type),
-				Major: hostDevice.Major,
-				Minor: hostDevice.Minor,
-				UID:   &hostDevice.Uid,
-				GID:   &hostDevice.Gid,
-			}
-			if hostDevice.Major == 0 && hostDevice.Minor == 0 {
-				// Invalid device, most likely a symbolic link, skip it.
-				continue
-			}
-			specgen.AddDevice(rd)
-		}
-		sp.Linux.Resources.Devices = []rspec.LinuxDeviceCgroup{
-			{
-				Allow:  true,
-				Access: "rwm",
-			},
-		}
-		return nil
-	}
-	for _, device := range containerConfig.GetDevices() {
-		path, err := resolveSymbolicLink(device.HostPath)
-		if err != nil {
-			return err
-		}
-		dev, err := devices.DeviceFromPath(path, device.Permissions)
-		// if there was no error, return the device
-		if err == nil {
-			rd := rspec.LinuxDevice{
-				Path:  device.ContainerPath,
-				Type:  string(dev.Type),
-				Major: dev.Major,
-				Minor: dev.Minor,
-				UID:   &dev.Uid,
-				GID:   &dev.Gid,
-			}
-			specgen.AddDevice(rd)
-			sp.Linux.Resources.Devices = append(sp.Linux.Resources.Devices, rspec.LinuxDeviceCgroup{
-				Allow:  true,
-				Type:   string(dev.Type),
-				Major:  &dev.Major,
-				Minor:  &dev.Minor,
-				Access: dev.Permissions,
-			})
-			continue
-		}
-		// if the device is not a device node
-		// try to see if it's a directory holding many devices
-		if err == devices.ErrNotADevice {
-
-			// check if it is a directory
-			if src, e := os.Stat(path); e == nil && src.IsDir() {
-
-				// mount the internal devices recursively
-				filepath.Walk(path, func(dpath string, f os.FileInfo, e error) error {
-					childDevice, e := devices.DeviceFromPath(dpath, device.Permissions)
-					if e != nil {
-						// ignore the device
-						return nil
-					}
-					cPath := strings.Replace(dpath, path, device.ContainerPath, 1)
-					rd := rspec.LinuxDevice{
-						Path:  cPath,
-						Type:  string(childDevice.Type),
-						Major: childDevice.Major,
-						Minor: childDevice.Minor,
-						UID:   &childDevice.Uid,
-						GID:   &childDevice.Gid,
-					}
-					specgen.AddDevice(rd)
-					sp.Linux.Resources.Devices = append(sp.Linux.Resources.Devices, rspec.LinuxDeviceCgroup{
-						Allow:  true,
-						Type:   string(childDevice.Type),
-						Major:  &childDevice.Major,
-						Minor:  &childDevice.Minor,
-						Access: childDevice.Permissions,
-					})
-
-					return nil
-				})
-			}
-		}
-	}
-	return nil
+	return addDevicesPlatform(sb, containerConfig, specgen)
 }
 
 // buildOCIProcessArgs build an OCI compatible process arguments slice.
@@ -449,7 +355,7 @@ func setupContainerUser(specgen *generate.Generator, rootfs string, sc *pb.Linux
 		containerUser := ""
 		// Case 1: run as user is set by kubelet
 		if sc.GetRunAsUser() != nil {
-			containerUser = strconv.FormatInt(sc.GetRunAsUser().Value, 10)
+			containerUser = strconv.FormatInt(sc.GetRunAsUser().GetValue(), 10)
 		} else {
 			// Case 2: run as username is set by kubelet
 			userName := sc.GetRunAsUsername()
@@ -600,7 +506,7 @@ func hostNetwork(containerConfig *pb.ContainerConfig) bool {
 		return false
 	}
 
-	return securityContext.GetNamespaceOptions().HostNetwork
+	return securityContext.GetNamespaceOptions().GetNetwork() == pb.NamespaceMode_NODE
 }
 
 // ensureSaneLogPath is a hack to fix https://issues.k8s.io/44043 which causes
@@ -757,6 +663,18 @@ func (s *Server) setupOCIHooks(specgen *generate.Generator, sb *sandbox.Sandbox,
 			}
 		}
 		for _, annotationRegex := range hook.Annotations {
+			for _, annotation := range containerConfig.GetAnnotations() {
+				match, err := regexp.MatchString(annotationRegex, annotation)
+				if err != nil {
+					logrus.Errorf("Invalid regex %q:%q", annotationRegex, err)
+					continue
+				}
+				if match {
+					if err := addHook(hook); err != nil {
+						return err
+					}
+				}
+			}
 			for _, annotation := range sb.Annotations() {
 				match, err := regexp.MatchString(annotationRegex, annotation)
 				if err != nil {
@@ -773,7 +691,7 @@ func (s *Server) setupOCIHooks(specgen *generate.Generator, sb *sandbox.Sandbox,
 	}
 	return nil
 }
-func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox.Sandbox, SandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) {
+func (s *Server) createSandboxContainer(ctx context.Context, containerID string, containerName string, sb *sandbox.Sandbox, sandboxConfig *pb.PodSandboxConfig, containerConfig *pb.ContainerConfig) (*oci.Container, error) {
 	if sb == nil {
 		return nil, errors.New("createSandboxContainer needs a sandbox")
 	}
@@ -871,15 +789,19 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 
 	}
 
-	logPath := containerConfig.LogPath
+	logPath := containerConfig.GetLogPath()
+	sboxLogDir := sandboxConfig.GetLogDirectory()
+	if sboxLogDir == "" {
+		sboxLogDir = sb.LogDir()
+	}
 	if logPath == "" {
-		// TODO: Should we use sandboxConfig.GetLogDirectory() here?
-		logPath = filepath.Join(sb.LogDir(), containerID+".log")
+		logPath = filepath.Join(sboxLogDir, containerID+".log")
 	}
 	if !filepath.IsAbs(logPath) {
 		// XXX: It's not really clear what this should be versus the sbox logDirectory.
 		logrus.Warnf("requested logPath for ctr id %s is a relative path: %s", containerID, logPath)
-		logPath = filepath.Join(sb.LogDir(), logPath)
+		logPath = filepath.Join(sboxLogDir, logPath)
+		logrus.Warnf("logPath from relative path is now absolute: %s", logPath)
 	}
 
 	// Handle https://issues.k8s.io/44043
@@ -888,8 +810,8 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	}
 
 	logrus.WithFields(logrus.Fields{
-		"sbox.logdir": sb.LogDir(),
-		"ctr.logfile": containerConfig.LogPath,
+		"sbox.logdir": sboxLogDir,
+		"ctr.logfile": containerConfig.GetLogPath(),
 		"log_path":    logPath,
 	}).Debugf("setting container's log_path")
 
@@ -981,10 +903,10 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 		return nil, err
 	}
 
-	if containerConfig.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostPid() {
+	if containerConfig.GetLinux().GetSecurityContext().GetNamespaceOptions().GetPid() == pb.NamespaceMode_NODE {
 		// kubernetes PodSpec specify to use Host PID namespace
 		specgen.RemoveLinuxNamespace(string(rspec.PIDNamespace))
-	} else if s.config.EnableSharedPIDNamespace {
+	} else if containerConfig.GetLinux().GetSecurityContext().GetNamespaceOptions().GetPid() == pb.NamespaceMode_POD {
 		// share Pod PID namespace
 		pidNsPath := fmt.Sprintf("/proc/%d/ns/pid", podInfraState.Pid)
 		if err := specgen.AddOrReplaceLinuxNamespace(string(rspec.PIDNamespace), pidNsPath); err != nil {
@@ -1022,14 +944,24 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	}
 
 	// Get imageName and imageRef that are later requested in container status
-	status, err := s.StorageImageServer().ImageStatus(s.ImageContext(), images[0])
-	if err != nil {
-		return nil, err
+	var (
+		imgResult    *storage.ImageResult
+		imgResultErr error
+	)
+	for _, img := range images {
+		imgResult, imgResultErr = s.StorageImageServer().ImageStatus(s.ImageContext(), img)
+		if imgResultErr == nil {
+			break
+		}
 	}
-	imageName := status.Name
-	imageRef := status.ID
-	if len(status.RepoDigests) > 0 {
-		imageRef = status.RepoDigests[0]
+	if imgResultErr != nil {
+		return nil, imgResultErr
+	}
+
+	imageName := imgResult.Name
+	imageRef := imgResult.ID
+	if len(imgResult.RepoDigests) > 0 {
+		imageRef = imgResult.RepoDigests[0]
 	}
 
 	specgen.AddAnnotation(annotations.Image, image)
@@ -1051,7 +983,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 		options = []string{"ro"}
 	}
 	if sb.ResolvPath() != "" {
-		if err := label.Relabel(sb.ResolvPath(), mountLabel, true); err != nil && err != unix.ENOTSUP {
+		if err := label.Relabel(sb.ResolvPath(), mountLabel, true); err != nil && err != syscall.ENOTSUP {
 			return nil, err
 		}
 
@@ -1066,7 +998,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	}
 
 	if sb.HostnamePath() != "" {
-		if err := label.Relabel(sb.HostnamePath(), mountLabel, true); err != nil && err != unix.ENOTSUP {
+		if err := label.Relabel(sb.HostnamePath(), mountLabel, true); err != nil && err != syscall.ENOTSUP {
 			return nil, err
 		}
 
@@ -1079,8 +1011,17 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 		specgen.AddMount(mnt)
 	}
 
-	// Bind mount /etc/hosts for host networking containers
-	if hostNetwork(containerConfig) {
+	isInCRIMounts := func(dst string, mounts []*pb.Mount) bool {
+		for _, m := range mounts {
+			if m.ContainerPath == dst {
+				return true
+			}
+		}
+		return false
+	}
+
+	if !isInCRIMounts("/etc/hosts", containerConfig.GetMounts()) && hostNetwork(containerConfig) {
+		// Only bind mount for host netns and when CRI does not give us any hosts file
 		mnt = rspec.Mount{
 			Type:        "bind",
 			Source:      "/etc/hosts",
@@ -1138,7 +1079,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	attempt := metadata.Attempt
 	containerInfo, err := s.StorageRuntimeServer().CreateContainer(s.ImageContext(),
 		sb.Name(), sb.ID(),
-		image, status.ID,
+		image, imgResult.ID,
 		containerName, containerID,
 		metaname,
 		attempt,
@@ -1250,8 +1191,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	}
 
 	// Set up pids limit if pids cgroup is mounted
-	_, err = cgroups.FindCgroupMountpoint("pids")
-	if err == nil {
+	if findCgroupMountpoint("pids") == nil {
 		specgen.SetLinuxResourcesPidsLimit(s.config.PidsLimit)
 	}
 
@@ -1268,7 +1208,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 
 	crioAnnotations := specgen.Spec().Annotations
 
-	container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs(), labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
+	container, err := oci.NewContainer(containerID, containerName, containerInfo.RunDir, logPath, sb.NetNs().Path(), labels, crioAnnotations, kubeAnnotations, image, imageName, imageRef, metadata, sb.ID(), containerConfig.Tty, containerConfig.Stdin, containerConfig.StdinOnce, sb.Privileged(), sb.Trusted(), containerInfo.Dir, created, containerImageConfig.Config.StopSignal)
 	if err != nil {
 		return nil, err
 	}
@@ -1405,7 +1345,7 @@ func setupWorkingDirectory(rootfs, mountLabel, containerCwd string) error {
 		return err
 	}
 	if mountLabel != "" {
-		if err1 := label.Relabel(fp, mountLabel, true); err1 != nil && err1 != unix.ENOTSUP {
+		if err1 := label.Relabel(fp, mountLabel, true); err1 != nil && err1 != syscall.ENOTSUP {
 			return fmt.Errorf("relabel failed %s: %v", fp, err1)
 		}
 	}

server/container_create_linux.go

@@ -0,0 +1,118 @@
+// +build linux
+
+package server
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/kubernetes-incubator/cri-o/lib/sandbox"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/devices"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+)
+
+func findCgroupMountpoint(name string) error {
+	// Set up pids limit if pids cgroup is mounted
+	_, err := cgroups.FindCgroupMountpoint(name)
+	return err
+}
+
+func addDevicesPlatform(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
+	sp := specgen.Spec()
+	if containerConfig.GetLinux().GetSecurityContext().GetPrivileged() {
+		hostDevices, err := devices.HostDevices()
+		if err != nil {
+			return err
+		}
+		for _, hostDevice := range hostDevices {
+			rd := rspec.LinuxDevice{
+				Path:  hostDevice.Path,
+				Type:  string(hostDevice.Type),
+				Major: hostDevice.Major,
+				Minor: hostDevice.Minor,
+				UID:   &hostDevice.Uid,
+				GID:   &hostDevice.Gid,
+			}
+			if hostDevice.Major == 0 && hostDevice.Minor == 0 {
+				// Invalid device, most likely a symbolic link, skip it.
+				continue
+			}
+			specgen.AddDevice(rd)
+		}
+		sp.Linux.Resources.Devices = []rspec.LinuxDeviceCgroup{
+			{
+				Allow:  true,
+				Access: "rwm",
+			},
+		}
+		return nil
+	}
+	for _, device := range containerConfig.GetDevices() {
+		path, err := resolveSymbolicLink(device.HostPath)
+		if err != nil {
+			return err
+		}
+		dev, err := devices.DeviceFromPath(path, device.Permissions)
+		// if there was no error, return the device
+		if err == nil {
+			rd := rspec.LinuxDevice{
+				Path:  device.ContainerPath,
+				Type:  string(dev.Type),
+				Major: dev.Major,
+				Minor: dev.Minor,
+				UID:   &dev.Uid,
+				GID:   &dev.Gid,
+			}
+			specgen.AddDevice(rd)
+			sp.Linux.Resources.Devices = append(sp.Linux.Resources.Devices, rspec.LinuxDeviceCgroup{
+				Allow:  true,
+				Type:   string(dev.Type),
+				Major:  &dev.Major,
+				Minor:  &dev.Minor,
+				Access: dev.Permissions,
+			})
+			continue
+		}
+		// if the device is not a device node
+		// try to see if it's a directory holding many devices
+		if err == devices.ErrNotADevice {
+
+			// check if it is a directory
+			if src, e := os.Stat(path); e == nil && src.IsDir() {
+
+				// mount the internal devices recursively
+				filepath.Walk(path, func(dpath string, f os.FileInfo, e error) error {
+					childDevice, e := devices.DeviceFromPath(dpath, device.Permissions)
+					if e != nil {
+						// ignore the device
+						return nil
+					}
+					cPath := strings.Replace(dpath, path, device.ContainerPath, 1)
+					rd := rspec.LinuxDevice{
+						Path:  cPath,
+						Type:  string(childDevice.Type),
+						Major: childDevice.Major,
+						Minor: childDevice.Minor,
+						UID:   &childDevice.Uid,
+						GID:   &childDevice.Gid,
+					}
+					specgen.AddDevice(rd)
+					sp.Linux.Resources.Devices = append(sp.Linux.Resources.Devices, rspec.LinuxDeviceCgroup{
+						Allow:  true,
+						Type:   string(childDevice.Type),
+						Major:  &childDevice.Major,
+						Minor:  &childDevice.Minor,
+						Access: childDevice.Permissions,
+					})
+
+					return nil
+				})
+			}
+		}
+	}
+	return nil
+}

server/container_create_unsupported.go

@@ -0,0 +1,19 @@
+// +build !linux
+
+package server
+
+import (
+	"fmt"
+
+	"github.com/kubernetes-incubator/cri-o/lib/sandbox"
+	"github.com/opencontainers/runtime-tools/generate"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+)
+
+func findCgroupMountpoint(name string) error {
+	return fmt.Errorf("no cgroups on this platform")
+}
+
+func addDevicesPlatform(sb *sandbox.Sandbox, containerConfig *pb.ContainerConfig, specgen *generate.Generator) error {
+	return nil
+}

server/container_exec.go

@@ -12,7 +12,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	"k8s.io/client-go/tools/remotecommand"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/util/term"
 	utilexec "k8s.io/utils/exec"

server/container_execsync.go

@@ -7,7 +7,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // ExecSync runs a command in a container synchronously.

server/container_list.go

@@ -7,7 +7,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	"k8s.io/apimachinery/pkg/fields"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // filterContainer returns whether passed container matches filtering criteria
@@ -28,6 +28,42 @@ func filterContainer(c *pb.Container, filter *pb.ContainerFilter) bool {
 	return true
 }
 
+// filterContainerList applies a protobuf-defined filter to retrieve only intended containers. Not matching
+// the filter is not considered an error but will return an empty response.
+func (s *Server) filterContainerList(filter *pb.ContainerFilter, origCtrList []*oci.Container) []*oci.Container {
+	// Filter using container id and pod id first.
+	if filter.Id != "" {
+		id, err := s.CtrIDIndex().Get(filter.Id)
+		if err != nil {
+			// If we don't find a container ID with a filter, it should not
+			// be considered an error.  Log a warning and return an empty struct
+			logrus.Warn("unable to find container ID %s", filter.Id)
+			return []*oci.Container{}
+		}
+		c := s.ContainerServer.GetContainer(id)
+		if c != nil {
+			switch {
+			case filter.PodSandboxId == "":
+				return []*oci.Container{c}
+			case c.Sandbox() == filter.PodSandboxId:
+				return []*oci.Container{c}
+			default:
+				return []*oci.Container{}
+			}
+		}
+	} else {
+		if filter.PodSandboxId != "" {
+			pod := s.ContainerServer.GetSandbox(filter.PodSandboxId)
+			if pod == nil {
+				return []*oci.Container{}
+			}
+			return pod.Containers().List()
+		}
+	}
+	logrus.Debug("no filters were applied, returning full container list")
+	return origCtrList
+}
+
 // ListContainers lists all containers by filters.
 func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersRequest) (resp *pb.ListContainersResponse, err error) {
 	const operation = "list_containers"
@@ -45,39 +81,7 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque
 	}
 
 	if filter != nil {
-
-		// Filter using container id and pod id first.
-		if filter.Id != "" {
-			id, err := s.CtrIDIndex().Get(filter.Id)
-			if err != nil {
-				// If we don't find a container ID with a filter, it should not
-				// be considered an error.  Log a warning and return an empty struct
-				logrus.Warn("unable to find container ID %s", filter.Id)
-				return &pb.ListContainersResponse{}, nil
-			}
-			c := s.ContainerServer.GetContainer(id)
-			if c != nil {
-				if filter.PodSandboxId != "" {
-					if c.Sandbox() == filter.PodSandboxId {
-						ctrList = []*oci.Container{c}
-					} else {
-						ctrList = []*oci.Container{}
-					}
-
-				} else {
-					ctrList = []*oci.Container{c}
-				}
-			}
-		} else {
-			if filter.PodSandboxId != "" {
-				pod := s.ContainerServer.GetSandbox(filter.PodSandboxId)
-				if pod == nil {
-					ctrList = []*oci.Container{}
-				} else {
-					ctrList = pod.Containers().List()
-				}
-			}
-		}
+		ctrList = s.filterContainerList(filter, ctrList)
 	}
 
 	for _, ctr := range ctrList {

server/container_portforward.go

@@ -12,7 +12,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // PortForward prepares a streaming endpoint to forward ports from a PodSandbox.

server/container_remove.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // RemoveContainer removes the container. If the container is running, the container

server/container_reopen_log.go

@@ -0,0 +1,13 @@
+package server
+
+import (
+	"fmt"
+
+	"golang.org/x/net/context"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+)
+
+// ReopenContainerLog reopens the containers log file
+func (s *Server) ReopenContainerLog(ctx context.Context, in *pb.ReopenContainerLogRequest) (*pb.ReopenContainerLogResponse, error) {
+	return nil, fmt.Errorf("not yet implemented")
+}

server/container_start.go

@@ -7,7 +7,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // StartContainer starts the container.

server/container_stats.go

@@ -4,10 +4,32 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/kubernetes-incubator/cri-o/lib"
+	"github.com/kubernetes-incubator/cri-o/oci"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
+func buildContainerStats(stats *lib.ContainerStats, container *oci.Container) *pb.ContainerStats {
+	return &pb.ContainerStats{
+		Attributes: &pb.ContainerAttributes{
+			Id:          container.ID(),
+			Metadata:    container.Metadata(),
+			Labels:      container.Labels(),
+			Annotations: container.Annotations(),
+		},
+		Cpu: &pb.CpuUsage{
+			Timestamp:            stats.SystemNano,
+			UsageCoreNanoSeconds: &pb.UInt64Value{Value: stats.CPUNano},
+		},
+		Memory: &pb.MemoryUsage{
+			Timestamp:       stats.SystemNano,
+			WorkingSetBytes: &pb.UInt64Value{Value: stats.MemUsage},
+		},
+		WritableLayer: nil,
+	}
+}
+
 // ContainerStats returns stats of the container. If the container does not
 // exist, the call returns an error.
 func (s *Server) ContainerStats(ctx context.Context, req *pb.ContainerStatsRequest) (resp *pb.ContainerStatsResponse, err error) {
@@ -16,5 +38,16 @@ func (s *Server) ContainerStats(ctx context.Context, req *pb.ContainerStatsReque
 		recordOperation(operation, time.Now())
 		recordError(operation, err)
 	}()
-	return nil, fmt.Errorf("not implemented")
+
+	container := s.GetContainer(req.ContainerId)
+	if container == nil {
+		return nil, fmt.Errorf("invalid container")
+	}
+
+	stats, err := s.GetContainerStats(container, &lib.ContainerStats{})
+	if err != nil {
+		return nil, err
+	}
+
+	return &pb.ContainerStatsResponse{Stats: buildContainerStats(stats, container)}, nil
 }

server/container_stats_list.go

@@ -1,11 +1,12 @@
 package server
 
 import (
-	"fmt"
 	"time"
 
+	"github.com/kubernetes-incubator/cri-o/lib"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // ListContainerStats returns stats of all running containers.
@@ -15,5 +16,34 @@ func (s *Server) ListContainerStats(ctx context.Context, req *pb.ListContainerSt
 		recordOperation(operation, time.Now())
 		recordError(operation, err)
 	}()
-	return nil, fmt.Errorf("not implemented")
+
+	ctrList, err := s.ContainerServer.ListContainers()
+	if err != nil {
+		return nil, err
+	}
+	filter := req.GetFilter()
+	if filter != nil {
+		cFilter := &pb.ContainerFilter{
+			Id:            req.Filter.Id,
+			PodSandboxId:  req.Filter.PodSandboxId,
+			LabelSelector: req.Filter.LabelSelector,
+		}
+		ctrList = s.filterContainerList(cFilter, ctrList)
+	}
+
+	var allStats []*pb.ContainerStats
+
+	for _, container := range ctrList {
+		stats, err := s.GetContainerStats(container, &lib.ContainerStats{})
+		if err != nil {
+			logrus.Warn("unable to get stats for container %s", container.ID())
+			continue
+		}
+		response := buildContainerStats(stats, container)
+		allStats = append(allStats, response)
+	}
+
+	return &pb.ListContainerStatsResponse{
+		Stats: allStats,
+	}, nil
 }

server/container_status.go

@@ -7,7 +7,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 const (
@@ -99,6 +99,7 @@ func (s *Server) ContainerStatus(ctx context.Context, req *pb.ContainerStatusReq
 	}
 
 	resp.Status.State = rStatus
+	resp.Status.LogPath = c.LogPath()
 
 	logrus.Debugf("ContainerStatusResponse: %+v", resp)
 	return resp, nil

server/container_stop.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // StopContainer stops a running container with a grace period (i.e., timeout).

server/container_update_resources.go

@@ -9,7 +9,7 @@ import (
 	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // UpdateContainerResources updates ContainerConfig of the container.

server/container_updateruntimeconfig.go

@@ -4,7 +4,7 @@ import (
 	"time"
 
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // UpdateRuntimeConfig updates the configuration of a running container.

server/image_fs_info.go

@@ -1,13 +1,35 @@
 package server
 
 import (
-	"fmt"
+	"path"
 	"time"
 
+	"github.com/containers/storage"
+	crioStorage "github.com/kubernetes-incubator/cri-o/utils"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
+func getStorageFsInfo(store storage.Store) (*pb.FilesystemUsage, error) {
+	rootPath := store.GraphRoot()
+	storageDriver := store.GraphDriverName()
+	imagesPath := path.Join(rootPath, storageDriver+"-images")
+
+	bytesUsed, inodesUsed, err := crioStorage.GetDiskUsageStats(imagesPath)
+	if err != nil {
+		return nil, err
+	}
+
+	usage := pb.FilesystemUsage{
+		Timestamp:  time.Now().UnixNano(),
+		FsId:       &pb.FilesystemIdentifier{Mountpoint: imagesPath},
+		UsedBytes:  &pb.UInt64Value{Value: bytesUsed},
+		InodesUsed: &pb.UInt64Value{Value: inodesUsed},
+	}
+
+	return &usage, nil
+}
+
 // ImageFsInfo returns information of the filesystem that is used to store images.
 func (s *Server) ImageFsInfo(ctx context.Context, req *pb.ImageFsInfoRequest) (resp *pb.ImageFsInfoResponse, err error) {
 	const operation = "image_fs_info"
@@ -16,5 +38,14 @@ func (s *Server) ImageFsInfo(ctx context.Context, req *pb.ImageFsInfoRequest) (r
 		recordError(operation, err)
 	}()
 
-	return nil, fmt.Errorf("not implemented")
+	store := s.StorageImageServer().GetStore()
+	fsUsage, err := getStorageFsInfo(store)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &pb.ImageFsInfoResponse{
+		ImageFilesystems: []*pb.FilesystemUsage{fsUsage},
+	}, nil
 }

server/image_list.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // ListImages lists existing images.
@@ -31,20 +31,20 @@ func (s *Server) ListImages(ctx context.Context, req *pb.ListImagesRequest) (res
 	}
 	resp = &pb.ListImagesResponse{}
 	for _, result := range results {
-		if result.Size != nil {
-			resp.Images = append(resp.Images, &pb.Image{
-				Id:          result.ID,
-				RepoTags:    result.RepoTags,
-				RepoDigests: result.RepoDigests,
-				Size_:       *result.Size,
-			})
-		} else {
-			resp.Images = append(resp.Images, &pb.Image{
-				Id:          result.ID,
-				RepoTags:    result.RepoTags,
-				RepoDigests: result.RepoDigests,
-			})
+		resImg := &pb.Image{
+			Id:          result.ID,
+			RepoTags:    result.RepoTags,
+			RepoDigests: result.RepoDigests,
 		}
+		uid, username := getUserFromImage(result.User)
+		if uid != nil {
+			resImg.Uid = &pb.Int64Value{Value: *uid}
+		}
+		resImg.Username = username
+		if result.Size != nil {
+			resImg.Size_ = *result.Size
+		}
+		resp.Images = append(resp.Images, resImg)
 	}
 	logrus.Debugf("ListImagesResponse: %+v", resp)
 	return resp, nil

server/image_pull.go

@@ -10,7 +10,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/pkg/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // PullImage pulls a image with authentication config.

server/image_remove.go

@@ -7,7 +7,7 @@ import (
 	"github.com/kubernetes-incubator/cri-o/pkg/storage"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // RemoveImage removes the image.

server/image_status.go

@@ -2,6 +2,8 @@ package server
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/containers/storage"
@@ -9,7 +11,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // ImageStatus returns the status of the image.
@@ -37,23 +39,62 @@ func (s *Server) ImageStatus(ctx context.Context, req *pb.ImageStatusRequest) (r
 			return nil, err
 		}
 	}
-	// match just the first registry as that's what kube meant
-	image = images[0]
-	status, err := s.StorageImageServer().ImageStatus(s.ImageContext(), image)
-	if err != nil {
-		if errors.Cause(err) == storage.ErrImageUnknown {
-			return &pb.ImageStatusResponse{}, nil
+	var (
+		notfound bool
+		lastErr  error
+	)
+	for _, image := range images {
+		status, err := s.StorageImageServer().ImageStatus(s.ImageContext(), image)
+		if err != nil {
+			if errors.Cause(err) == storage.ErrImageUnknown {
+				logrus.Warnf("imageStatus: can't find %s", image)
+				notfound = true
+				continue
+			}
+			logrus.Warnf("imageStatus: error getting status from %s: %v", image, err)
+			lastErr = err
+			continue
 		}
-		return nil, err
+		resp = &pb.ImageStatusResponse{
+			Image: &pb.Image{
+				Id:          status.ID,
+				RepoTags:    status.RepoTags,
+				RepoDigests: status.RepoDigests,
+				Size_:       *status.Size,
+			},
+		}
+		uid, username := getUserFromImage(status.User)
+		if uid != nil {
+			resp.Image.Uid = &pb.Int64Value{Value: *uid}
+		}
+		resp.Image.Username = username
+		break
 	}
-	resp = &pb.ImageStatusResponse{
-		Image: &pb.Image{
-			Id:          status.ID,
-			RepoTags:    status.RepoTags,
-			RepoDigests: status.RepoDigests,
-			Size_:       *status.Size,
-		},
+	if lastErr != nil && resp == nil {
+		return nil, lastErr
+	}
+	if notfound && resp == nil {
+		return &pb.ImageStatusResponse{}, nil
 	}
 	logrus.Debugf("ImageStatusResponse: %+v", resp)
 	return resp, nil
 }
+
+// getUserFromImage gets uid or user name of the image user.
+// If user is numeric, it will be treated as uid; or else, it is treated as user name.
+func getUserFromImage(user string) (*int64, string) {
+	// return both empty if user is not specified in the image.
+	if user == "" {
+		return nil, ""
+	}
+	// split instances where the id may contain user:group
+	user = strings.Split(user, ":")[0]
+	// user could be either uid or user name. Try to interpret as numeric uid.
+	uid, err := strconv.ParseInt(user, 10, 64)
+	if err != nil {
+		// If user is non numeric, assume it's user name.
+		return nil, user
+	}
+	// If user is a numeric uid.
+	return &uid, ""
+}

server/inspect_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/kubernetes-incubator/cri-o/lib"

server/naming.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 
 	"github.com/docker/docker/pkg/stringid"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 const (

server/runtime_status.go

@@ -4,7 +4,7 @@ import (
 	"time"
 
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // Status returns the status of the runtime

server/sandbox_list.go

@@ -8,7 +8,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	"k8s.io/apimachinery/pkg/fields"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // filterSandbox returns whether passed container matches filtering criteria

server/sandbox_remove.go

@@ -11,7 +11,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // RemovePodSandbox deletes the sandbox. If there are any running containers in the
@@ -48,10 +48,14 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
 		if !sb.Stopped() {
 			cState := s.Runtime().ContainerStatus(c)
 			if cState.Status == oci.ContainerStateCreated || cState.Status == oci.ContainerStateRunning {
-				if err := s.Runtime().StopContainer(ctx, c, 10); err != nil {
+				timeout := int64(10)
+				if err := s.Runtime().StopContainer(ctx, c, timeout); err != nil {
 					// Assume container is already stopped
 					logrus.Warnf("failed to stop container %s: %v", c.Name(), err)
 				}
+				if err := s.Runtime().WaitContainerStateStopped(ctx, c, timeout); err != nil {
+					return nil, fmt.Errorf("failed to get container 'stopped' status %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
+				}
 			}
 		}
 

server/sandbox_run.go

@@ -1,33 +1,17 @@
 package server
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
 	"regexp"
-	"strconv"
-	"strings"
-	"time"
 
-	"github.com/containers/storage"
-	"github.com/kubernetes-incubator/cri-o/lib/sandbox"
 	"github.com/kubernetes-incubator/cri-o/oci"
 	"github.com/kubernetes-incubator/cri-o/pkg/annotations"
-	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/opencontainers/runtime-tools/generate"
 	"github.com/opencontainers/selinux/go-selinux/label"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	"golang.org/x/sys/unix"
 	"k8s.io/api/core/v1"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
-	"k8s.io/kubernetes/pkg/kubelet/leaky"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	"k8s.io/kubernetes/pkg/kubelet/network/hostport"
-	"k8s.io/kubernetes/pkg/kubelet/types"
 )
 
 const (
@@ -57,9 +41,9 @@ func (s *Server) privilegedSandbox(req *pb.RunPodSandboxRequest) bool {
 		return false
 	}
 
-	if namespaceOptions.HostNetwork ||
-		namespaceOptions.HostPid ||
-		namespaceOptions.HostIpc {
+	if namespaceOptions.GetNetwork() == pb.NamespaceMode_NODE ||
+		namespaceOptions.GetPid() == pb.NamespaceMode_NODE ||
+		namespaceOptions.GetIpc() == pb.NamespaceMode_NODE {
 		return true
 	}
 
@@ -92,464 +76,8 @@ var (
 
 // RunPodSandbox creates and runs a pod-level sandbox.
 func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (resp *pb.RunPodSandboxResponse, err error) {
-	const operation = "run_pod_sandbox"
-	defer func() {
-		recordOperation(operation, time.Now())
-		recordError(operation, err)
-	}()
-
-	s.updateLock.RLock()
-	defer s.updateLock.RUnlock()
-
-	if req.GetConfig().GetMetadata() == nil {
-		return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Metadata is nil")
-	}
-
-	logrus.Debugf("RunPodSandboxRequest %+v", req)
-	var processLabel, mountLabel, resolvPath string
-	// process req.Name
-	kubeName := req.GetConfig().GetMetadata().GetName()
-	if kubeName == "" {
-		return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty")
-	}
-
-	namespace := req.GetConfig().GetMetadata().GetNamespace()
-	attempt := req.GetConfig().GetMetadata().GetAttempt()
-
-	id, name, err := s.generatePodIDandName(req.GetConfig())
-	if err != nil {
-		if strings.Contains(err.Error(), "already reserved for pod") {
-			matches := conflictRE.FindStringSubmatch(err.Error())
-			if len(matches) != 2 {
-				return nil, err
-			}
-			dupID := matches[1]
-			if _, err := s.StopPodSandbox(ctx, &pb.StopPodSandboxRequest{PodSandboxId: dupID}); err != nil {
-				return nil, err
-			}
-			if _, err := s.RemovePodSandbox(ctx, &pb.RemovePodSandboxRequest{PodSandboxId: dupID}); err != nil {
-				return nil, err
-			}
-			id, name, err = s.generatePodIDandName(req.GetConfig())
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			return nil, err
-		}
-	}
-
-	defer func() {
-		if err != nil {
-			s.ReleasePodName(name)
-		}
-	}()
-
-	_, containerName, err := s.generateContainerIDandNameForSandbox(req.GetConfig())
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		if err != nil {
-			s.ReleaseContainerName(containerName)
-		}
-	}()
-
-	podContainer, err := s.StorageRuntimeServer().CreatePodSandbox(s.ImageContext(),
-		name, id,
-		s.config.PauseImage, "",
-		containerName,
-		req.GetConfig().GetMetadata().GetName(),
-		req.GetConfig().GetMetadata().GetUid(),
-		namespace,
-		attempt,
-		nil)
-	if errors.Cause(err) == storage.ErrDuplicateName {
-		return nil, fmt.Errorf("pod sandbox with name %q already exists", name)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("error creating pod sandbox with name %q: %v", name, err)
-	}
-	defer func() {
-		if err != nil {
-			if err2 := s.StorageRuntimeServer().RemovePodSandbox(id); err2 != nil {
-				logrus.Warnf("couldn't cleanup pod sandbox %q: %v", id, err2)
-			}
-		}
-	}()
-
-	// TODO: factor generating/updating the spec into something other projects can vendor
-
-	// creates a spec Generator with the default spec.
-	g := generate.New()
-
-	// setup defaults for the pod sandbox
-	g.SetRootReadonly(true)
-	if s.config.PauseCommand == "" {
-		if podContainer.Config != nil {
-			g.SetProcessArgs(podContainer.Config.Config.Cmd)
-		} else {
-			g.SetProcessArgs([]string{sandbox.PodInfraCommand})
-		}
-	} else {
-		g.SetProcessArgs([]string{s.config.PauseCommand})
-	}
-
-	// set DNS options
-	if req.GetConfig().GetDnsConfig() != nil {
-		dnsServers := req.GetConfig().GetDnsConfig().Servers
-		dnsSearches := req.GetConfig().GetDnsConfig().Searches
-		dnsOptions := req.GetConfig().GetDnsConfig().Options
-		resolvPath = fmt.Sprintf("%s/resolv.conf", podContainer.RunDir)
-		err = parseDNSOptions(dnsServers, dnsSearches, dnsOptions, resolvPath)
-		if err != nil {
-			err1 := removeFile(resolvPath)
-			if err1 != nil {
-				err = err1
-				return nil, fmt.Errorf("%v; failed to remove %s: %v", err, resolvPath, err1)
-			}
-			return nil, err
-		}
-		if err := label.Relabel(resolvPath, mountLabel, true); err != nil && err != unix.ENOTSUP {
-			return nil, err
-		}
-		mnt := runtimespec.Mount{
-			Type:        "bind",
-			Source:      resolvPath,
-			Destination: "/etc/resolv.conf",
-			Options:     []string{"ro", "bind"},
-		}
-		g.AddMount(mnt)
-	}
-
-	// add metadata
-	metadata := req.GetConfig().GetMetadata()
-	metadataJSON, err := json.Marshal(metadata)
-	if err != nil {
-		return nil, err
-	}
-
-	// add labels
-	labels := req.GetConfig().GetLabels()
-
-	if err := validateLabels(labels); err != nil {
-		return nil, err
-	}
-
-	// Add special container name label for the infra container
-	labelsJSON := []byte{}
-	if labels != nil {
-		labels[types.KubernetesContainerNameLabel] = leaky.PodInfraContainerName
-		labelsJSON, err = json.Marshal(labels)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// add annotations
-	kubeAnnotations := req.GetConfig().GetAnnotations()
-	kubeAnnotationsJSON, err := json.Marshal(kubeAnnotations)
-	if err != nil {
-		return nil, err
-	}
-
-	// set log directory
-	logDir := req.GetConfig().LogDirectory
-	if logDir == "" {
-		logDir = filepath.Join(s.config.LogDir, id)
-	}
-	if err = os.MkdirAll(logDir, 0700); err != nil {
-		return nil, err
-	}
-	// This should always be absolute from k8s.
-	if !filepath.IsAbs(logDir) {
-		return nil, fmt.Errorf("requested logDir for sbox id %s is a relative path: %s", id, logDir)
-	}
-
-	privileged := s.privilegedSandbox(req)
-
-	securityContext := req.GetConfig().GetLinux().GetSecurityContext()
-	if securityContext == nil {
-		logrus.Warn("no security context found in config.")
-	}
-
-	processLabel, mountLabel, err = getSELinuxLabels(securityContext.GetSelinuxOptions(), privileged)
-	if err != nil {
-		return nil, err
-	}
-
-	// Don't use SELinux separation with Host Pid or IPC Namespace or privileged.
-	if securityContext.GetNamespaceOptions().GetHostPid() || securityContext.GetNamespaceOptions().GetHostIpc() {
-		processLabel, mountLabel = "", ""
-	}
-	g.SetProcessSelinuxLabel(processLabel)
-	g.SetLinuxMountLabel(mountLabel)
-
-	// create shm mount for the pod containers.
-	var shmPath string
-	if securityContext.GetNamespaceOptions().GetHostIpc() {
-		shmPath = "/dev/shm"
-	} else {
-		shmPath, err = setupShm(podContainer.RunDir, mountLabel)
-		if err != nil {
-			return nil, err
-		}
-		defer func() {
-			if err != nil {
-				if err2 := unix.Unmount(shmPath, unix.MNT_DETACH); err2 != nil {
-					logrus.Warnf("failed to unmount shm for pod: %v", err2)
-				}
-			}
-		}()
-	}
-
-	err = s.setPodSandboxMountLabel(id, mountLabel)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = s.CtrIDIndex().Add(id); err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		if err != nil {
-			if err2 := s.CtrIDIndex().Delete(id); err2 != nil {
-				logrus.Warnf("couldn't delete ctr id %s from idIndex", id)
-			}
-		}
-	}()
-
-	// set log path inside log directory
-	logPath := filepath.Join(logDir, id+".log")
-
-	// Handle https://issues.k8s.io/44043
-	if err := ensureSaneLogPath(logPath); err != nil {
-		return nil, err
-	}
-
-	hostNetwork := securityContext.GetNamespaceOptions().GetHostNetwork()
-
-	hostname, err := getHostname(id, req.GetConfig().Hostname, hostNetwork)
-	if err != nil {
-		return nil, err
-	}
-	g.SetHostname(hostname)
-
-	trusted := s.trustedSandbox(req)
-	g.AddAnnotation(annotations.Metadata, string(metadataJSON))
-	g.AddAnnotation(annotations.Labels, string(labelsJSON))
-	g.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON))
-	g.AddAnnotation(annotations.LogPath, logPath)
-	g.AddAnnotation(annotations.Name, name)
-	g.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox)
-	g.AddAnnotation(annotations.SandboxID, id)
-	g.AddAnnotation(annotations.ContainerName, containerName)
-	g.AddAnnotation(annotations.ContainerID, id)
-	g.AddAnnotation(annotations.ShmPath, shmPath)
-	g.AddAnnotation(annotations.PrivilegedRuntime, fmt.Sprintf("%v", privileged))
-	g.AddAnnotation(annotations.TrustedSandbox, fmt.Sprintf("%v", trusted))
-	g.AddAnnotation(annotations.ResolvPath, resolvPath)
-	g.AddAnnotation(annotations.HostName, hostname)
-	g.AddAnnotation(annotations.KubeName, kubeName)
-	if podContainer.Config.Config.StopSignal != "" {
-		// this key is defined in image-spec conversion document at https://github.com/opencontainers/image-spec/pull/492/files#diff-8aafbe2c3690162540381b8cdb157112R57
-		g.AddAnnotation("org.opencontainers.image.stopSignal", podContainer.Config.Config.StopSignal)
-	}
-
-	created := time.Now()
-	g.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano))
-
-	portMappings := convertPortMappings(req.GetConfig().GetPortMappings())
-
-	// setup cgroup settings
-	cgroupParent := req.GetConfig().GetLinux().GetCgroupParent()
-	if cgroupParent != "" {
-		if s.config.CgroupManager == oci.SystemdCgroupsManager {
-			if len(cgroupParent) <= 6 || !strings.HasSuffix(path.Base(cgroupParent), ".slice") {
-				return nil, fmt.Errorf("cri-o configured with systemd cgroup manager, but did not receive slice as parent: %s", cgroupParent)
-			}
-			cgPath, err := convertCgroupFsNameToSystemd(cgroupParent)
-			if err != nil {
-				return nil, err
-			}
-			g.SetLinuxCgroupsPath(cgPath + ":" + "crio" + ":" + id)
-			cgroupParent = cgPath
-		} else {
-			if strings.HasSuffix(path.Base(cgroupParent), ".slice") {
-				return nil, fmt.Errorf("cri-o configured with cgroupfs cgroup manager, but received systemd slice as parent: %s", cgroupParent)
-			}
-			cgPath := filepath.Join(cgroupParent, scopePrefix+"-"+id)
-			g.SetLinuxCgroupsPath(cgPath)
-		}
-	}
-
-	sb, err := sandbox.New(id, namespace, name, kubeName, logDir, labels, kubeAnnotations, processLabel, mountLabel, metadata, shmPath, cgroupParent, privileged, trusted, resolvPath, hostname, portMappings)
-	if err != nil {
-		return nil, err
-	}
-
-	s.addSandbox(sb)
-	defer func() {
-		if err != nil {
-			s.removeSandbox(id)
-		}
-	}()
-
-	if err = s.PodIDIndex().Add(id); err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		if err != nil {
-			if err := s.PodIDIndex().Delete(id); err != nil {
-				logrus.Warnf("couldn't delete pod id %s from idIndex", id)
-			}
-		}
-	}()
-
-	for k, v := range kubeAnnotations {
-		g.AddAnnotation(k, v)
-	}
-	for k, v := range labels {
-		g.AddAnnotation(k, v)
-	}
-
-	// extract linux sysctls from annotations and pass down to oci runtime
-	for key, value := range req.GetConfig().GetLinux().GetSysctls() {
-		g.AddLinuxSysctl(key, value)
-	}
-
-	// Set OOM score adjust of the infra container to be very low
-	// so it doesn't get killed.
-	g.SetProcessOOMScoreAdj(PodInfraOOMAdj)
-
-	g.SetLinuxResourcesCPUShares(PodInfraCPUshares)
-
-	// set up namespaces
-	if hostNetwork {
-		err = g.RemoveLinuxNamespace(string(runtimespec.NetworkNamespace))
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		// Create the sandbox network namespace
-		if err = sb.NetNsCreate(); err != nil {
-			return nil, err
-		}
-
-		defer func() {
-			if err == nil {
-				return
-			}
-
-			if netnsErr := sb.NetNsRemove(); netnsErr != nil {
-				logrus.Warnf("Failed to remove networking namespace: %v", netnsErr)
-			}
-		}()
-
-		// Pass the created namespace path to the runtime
-		err = g.AddOrReplaceLinuxNamespace(string(runtimespec.NetworkNamespace), sb.NetNsPath())
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if securityContext.GetNamespaceOptions().GetHostPid() {
-		err = g.RemoveLinuxNamespace(string(runtimespec.PIDNamespace))
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if securityContext.GetNamespaceOptions().GetHostIpc() {
-		err = g.RemoveLinuxNamespace(string(runtimespec.IPCNamespace))
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if !s.seccompEnabled {
-		g.Spec().Linux.Seccomp = nil
-	}
-
-	saveOptions := generate.ExportOptions{}
-	mountPoint, err := s.StorageRuntimeServer().StartContainer(id)
-	if err != nil {
-		return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.Name(), id, err)
-	}
-	g.AddAnnotation(annotations.MountPoint, mountPoint)
-	g.SetRootPath(mountPoint)
-
-	hostnamePath := fmt.Sprintf("%s/hostname", podContainer.RunDir)
-	if err := ioutil.WriteFile(hostnamePath, []byte(hostname+"\n"), 0644); err != nil {
-		return nil, err
-	}
-	if err := label.Relabel(hostnamePath, mountLabel, true); err != nil && err != unix.ENOTSUP {
-		return nil, err
-	}
-	mnt := runtimespec.Mount{
-		Type:        "bind",
-		Source:      hostnamePath,
-		Destination: "/etc/hostname",
-		Options:     []string{"ro", "bind"},
-	}
-	g.AddMount(mnt)
-	g.AddAnnotation(annotations.HostnamePath, hostnamePath)
-	sb.AddHostnamePath(hostnamePath)
-
-	container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
-	if err != nil {
-		return nil, err
-	}
-	container.SetSpec(g.Spec())
-	container.SetMountPoint(mountPoint)
-
-	sb.SetInfraContainer(container)
-
-	var ip string
-	ip, err = s.networkStart(hostNetwork, sb)
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if err != nil {
-			s.networkStop(hostNetwork, sb)
-		}
-	}()
-
-	g.AddAnnotation(annotations.IP, ip)
-	sb.AddIP(ip)
-
-	spp := req.GetConfig().GetLinux().GetSecurityContext().GetSeccompProfilePath()
-	g.AddAnnotation(annotations.SeccompProfilePath, spp)
-	sb.SetSeccompProfilePath(spp)
-	if !privileged {
-		if err = s.setupSeccomp(&g, spp); err != nil {
-			return nil, err
-		}
-	}
-
-	err = g.SaveToFile(filepath.Join(podContainer.Dir, "config.json"), saveOptions)
-	if err != nil {
-		return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.Name(), id, err)
-	}
-	if err = g.SaveToFile(filepath.Join(podContainer.RunDir, "config.json"), saveOptions); err != nil {
-		return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.Name(), id, err)
-	}
-
-	if err = s.runContainer(container, sb.CgroupParent()); err != nil {
-		return nil, err
-	}
-
-	s.addInfraContainer(container)
-
-	s.ContainerStateToDisk(container)
-
-	resp = &pb.RunPodSandboxResponse{PodSandboxId: id}
-	logrus.Debugf("RunPodSandboxResponse: %+v", resp)
-	return resp, nil
+	// platform dependent call
+	return s.runPodSandbox(ctx, req)
 }
 
 func convertPortMappings(in []*pb.PortMapping) []*hostport.PortMapping {
@@ -616,19 +144,6 @@ func getSELinuxLabels(selinuxOptions *pb.SELinuxOption, privileged bool) (proces
 	return label.InitLabels(labels)
 }
 
-func setupShm(podSandboxRunDir, mountLabel string) (shmPath string, err error) {
-	shmPath = filepath.Join(podSandboxRunDir, "shm")
-	if err = os.Mkdir(shmPath, 0700); err != nil {
-		return "", err
-	}
-	shmOptions := "mode=1777,size=" + strconv.Itoa(sandbox.DefaultShmSize)
-	if err = unix.Mount("shm", shmPath, "tmpfs", unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV,
-		label.FormatMountLabel(shmOptions, mountLabel)); err != nil {
-		return "", fmt.Errorf("failed to mount shm tmpfs for pod: %v", err)
-	}
-	return shmPath, nil
-}
-
 // convertCgroupFsNameToSystemd converts an expanded cgroupfs name to its systemd name.
 // For example, it will convert test.slice/test-a.slice/test-a-b.slice to become test-a-b.slice
 // NOTE: this is public right now to allow its usage in dockermanager and dockershim, ideally both those

server/sandbox_run_linux.go

@@ -0,0 +1,519 @@
+// +build linux
+
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/containers/storage"
+	"github.com/kubernetes-incubator/cri-o/lib/sandbox"
+	"github.com/kubernetes-incubator/cri-o/oci"
+	"github.com/kubernetes-incubator/cri-o/pkg/annotations"
+	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
+	"golang.org/x/sys/unix"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+	"k8s.io/kubernetes/pkg/kubelet/leaky"
+	"k8s.io/kubernetes/pkg/kubelet/types"
+)
+
+func (s *Server) runPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (resp *pb.RunPodSandboxResponse, err error) {
+	const operation = "run_pod_sandbox"
+	defer func() {
+		recordOperation(operation, time.Now())
+		recordError(operation, err)
+	}()
+
+	s.updateLock.RLock()
+	defer s.updateLock.RUnlock()
+
+	if req.GetConfig().GetMetadata() == nil {
+		return nil, fmt.Errorf("CreateContainerRequest.ContainerConfig.Metadata is nil")
+	}
+
+	logrus.Debugf("RunPodSandboxRequest %+v", req)
+	var processLabel, mountLabel, resolvPath string
+	// process req.Name
+	kubeName := req.GetConfig().GetMetadata().GetName()
+	if kubeName == "" {
+		return nil, fmt.Errorf("PodSandboxConfig.Name should not be empty")
+	}
+
+	namespace := req.GetConfig().GetMetadata().GetNamespace()
+	attempt := req.GetConfig().GetMetadata().GetAttempt()
+
+	id, name, err := s.generatePodIDandName(req.GetConfig())
+	if err != nil {
+		if strings.Contains(err.Error(), "already reserved for pod") {
+			matches := conflictRE.FindStringSubmatch(err.Error())
+			if len(matches) != 2 {
+				return nil, err
+			}
+			dupID := matches[1]
+			if _, err := s.StopPodSandbox(ctx, &pb.StopPodSandboxRequest{PodSandboxId: dupID}); err != nil {
+				return nil, err
+			}
+			if _, err := s.RemovePodSandbox(ctx, &pb.RemovePodSandboxRequest{PodSandboxId: dupID}); err != nil {
+				return nil, err
+			}
+			id, name, err = s.generatePodIDandName(req.GetConfig())
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	defer func() {
+		if err != nil {
+			s.ReleasePodName(name)
+		}
+	}()
+
+	_, containerName, err := s.generateContainerIDandNameForSandbox(req.GetConfig())
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			s.ReleaseContainerName(containerName)
+		}
+	}()
+
+	podContainer, err := s.StorageRuntimeServer().CreatePodSandbox(s.ImageContext(),
+		name, id,
+		s.config.PauseImage, "",
+		containerName,
+		req.GetConfig().GetMetadata().GetName(),
+		req.GetConfig().GetMetadata().GetUid(),
+		namespace,
+		attempt,
+		nil)
+	if errors.Cause(err) == storage.ErrDuplicateName {
+		return nil, fmt.Errorf("pod sandbox with name %q already exists", name)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("error creating pod sandbox with name %q: %v", name, err)
+	}
+	defer func() {
+		if err != nil {
+			if err2 := s.StorageRuntimeServer().RemovePodSandbox(id); err2 != nil {
+				logrus.Warnf("couldn't cleanup pod sandbox %q: %v", id, err2)
+			}
+		}
+	}()
+
+	// TODO: factor generating/updating the spec into something other projects can vendor
+
+	// creates a spec Generator with the default spec.
+	g := generate.New()
+
+	// setup defaults for the pod sandbox
+	g.SetRootReadonly(true)
+	if s.config.PauseCommand == "" {
+		if podContainer.Config != nil {
+			g.SetProcessArgs(podContainer.Config.Config.Cmd)
+		} else {
+			g.SetProcessArgs([]string{sandbox.PodInfraCommand})
+		}
+	} else {
+		g.SetProcessArgs([]string{s.config.PauseCommand})
+	}
+
+	// set DNS options
+	if req.GetConfig().GetDnsConfig() != nil {
+		dnsServers := req.GetConfig().GetDnsConfig().Servers
+		dnsSearches := req.GetConfig().GetDnsConfig().Searches
+		dnsOptions := req.GetConfig().GetDnsConfig().Options
+		resolvPath = fmt.Sprintf("%s/resolv.conf", podContainer.RunDir)
+		err = parseDNSOptions(dnsServers, dnsSearches, dnsOptions, resolvPath)
+		if err != nil {
+			err1 := removeFile(resolvPath)
+			if err1 != nil {
+				err = err1
+				return nil, fmt.Errorf("%v; failed to remove %s: %v", err, resolvPath, err1)
+			}
+			return nil, err
+		}
+		if err := label.Relabel(resolvPath, mountLabel, true); err != nil && err != unix.ENOTSUP {
+			return nil, err
+		}
+		mnt := runtimespec.Mount{
+			Type:        "bind",
+			Source:      resolvPath,
+			Destination: "/etc/resolv.conf",
+			Options:     []string{"ro", "bind"},
+		}
+		g.AddMount(mnt)
+	}
+
+	// add metadata
+	metadata := req.GetConfig().GetMetadata()
+	metadataJSON, err := json.Marshal(metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	// add labels
+	labels := req.GetConfig().GetLabels()
+
+	if err := validateLabels(labels); err != nil {
+		return nil, err
+	}
+
+	// Add special container name label for the infra container
+	labelsJSON := []byte{}
+	if labels != nil {
+		labels[types.KubernetesContainerNameLabel] = leaky.PodInfraContainerName
+		labelsJSON, err = json.Marshal(labels)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// add annotations
+	kubeAnnotations := req.GetConfig().GetAnnotations()
+	kubeAnnotationsJSON, err := json.Marshal(kubeAnnotations)
+	if err != nil {
+		return nil, err
+	}
+
+	// set log directory
+	logDir := req.GetConfig().GetLogDirectory()
+	if logDir == "" {
+		logDir = filepath.Join(s.config.LogDir, id)
+	}
+	if err = os.MkdirAll(logDir, 0700); err != nil {
+		return nil, err
+	}
+	// This should always be absolute from k8s.
+	if !filepath.IsAbs(logDir) {
+		return nil, fmt.Errorf("requested logDir for sbox id %s is a relative path: %s", id, logDir)
+	}
+
+	privileged := s.privilegedSandbox(req)
+
+	securityContext := req.GetConfig().GetLinux().GetSecurityContext()
+	if securityContext == nil {
+		logrus.Warn("no security context found in config.")
+	}
+
+	nsOptsJSON, err := json.Marshal(securityContext.GetNamespaceOptions())
+	if err != nil {
+		return nil, err
+	}
+
+	processLabel, mountLabel, err = getSELinuxLabels(securityContext.GetSelinuxOptions(), privileged)
+	if err != nil {
+		return nil, err
+	}
+
+	// Don't use SELinux separation with Host Pid or IPC Namespace or privileged.
+	if securityContext.GetNamespaceOptions().GetPid() == pb.NamespaceMode_NODE ||
+		securityContext.GetNamespaceOptions().GetIpc() == pb.NamespaceMode_NODE {
+		processLabel, mountLabel = "", ""
+	}
+	g.SetProcessSelinuxLabel(processLabel)
+	g.SetLinuxMountLabel(mountLabel)
+
+	// create shm mount for the pod containers.
+	var shmPath string
+	if securityContext.GetNamespaceOptions().GetIpc() == pb.NamespaceMode_NODE {
+		shmPath = "/dev/shm"
+	} else {
+		shmPath, err = setupShm(podContainer.RunDir, mountLabel)
+		if err != nil {
+			return nil, err
+		}
+		defer func() {
+			if err != nil {
+				if err2 := unix.Unmount(shmPath, unix.MNT_DETACH); err2 != nil {
+					logrus.Warnf("failed to unmount shm for pod: %v", err2)
+				}
+			}
+		}()
+	}
+
+	err = s.setPodSandboxMountLabel(id, mountLabel)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = s.CtrIDIndex().Add(id); err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			if err2 := s.CtrIDIndex().Delete(id); err2 != nil {
+				logrus.Warnf("couldn't delete ctr id %s from idIndex", id)
+			}
+		}
+	}()
+
+	// set log path inside log directory
+	logPath := filepath.Join(logDir, id+".log")
+
+	// Handle https://issues.k8s.io/44043
+	if err := ensureSaneLogPath(logPath); err != nil {
+		return nil, err
+	}
+
+	hostNetwork := securityContext.GetNamespaceOptions().GetNetwork() == pb.NamespaceMode_NODE
+
+	hostname, err := getHostname(id, req.GetConfig().Hostname, hostNetwork)
+	if err != nil {
+		return nil, err
+	}
+	g.SetHostname(hostname)
+
+	trusted := s.trustedSandbox(req)
+	g.AddAnnotation(annotations.Metadata, string(metadataJSON))
+	g.AddAnnotation(annotations.Labels, string(labelsJSON))
+	g.AddAnnotation(annotations.Annotations, string(kubeAnnotationsJSON))
+	g.AddAnnotation(annotations.LogPath, logPath)
+	g.AddAnnotation(annotations.Name, name)
+	g.AddAnnotation(annotations.Namespace, namespace)
+	g.AddAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox)
+	g.AddAnnotation(annotations.SandboxID, id)
+	g.AddAnnotation(annotations.ContainerName, containerName)
+	g.AddAnnotation(annotations.ContainerID, id)
+	g.AddAnnotation(annotations.ShmPath, shmPath)
+	g.AddAnnotation(annotations.PrivilegedRuntime, fmt.Sprintf("%v", privileged))
+	g.AddAnnotation(annotations.TrustedSandbox, fmt.Sprintf("%v", trusted))
+	g.AddAnnotation(annotations.ResolvPath, resolvPath)
+	g.AddAnnotation(annotations.HostName, hostname)
+	g.AddAnnotation(annotations.NamespaceOptions, string(nsOptsJSON))
+	g.AddAnnotation(annotations.KubeName, kubeName)
+	if podContainer.Config.Config.StopSignal != "" {
+		// this key is defined in image-spec conversion document at https://github.com/opencontainers/image-spec/pull/492/files#diff-8aafbe2c3690162540381b8cdb157112R57
+		g.AddAnnotation("org.opencontainers.image.stopSignal", podContainer.Config.Config.StopSignal)
+	}
+
+	created := time.Now()
+	g.AddAnnotation(annotations.Created, created.Format(time.RFC3339Nano))
+
+	portMappings := convertPortMappings(req.GetConfig().GetPortMappings())
+	portMappingsJSON, err := json.Marshal(portMappings)
+	if err != nil {
+		return nil, err
+	}
+	g.AddAnnotation(annotations.PortMappings, string(portMappingsJSON))
+
+	// setup cgroup settings
+	cgroupParent := req.GetConfig().GetLinux().GetCgroupParent()
+	if cgroupParent != "" {
+		if s.config.CgroupManager == oci.SystemdCgroupsManager {
+			if len(cgroupParent) <= 6 || !strings.HasSuffix(path.Base(cgroupParent), ".slice") {
+				return nil, fmt.Errorf("cri-o configured with systemd cgroup manager, but did not receive slice as parent: %s", cgroupParent)
+			}
+			cgPath, err := convertCgroupFsNameToSystemd(cgroupParent)
+			if err != nil {
+				return nil, err
+			}
+			g.SetLinuxCgroupsPath(cgPath + ":" + "crio" + ":" + id)
+			cgroupParent = cgPath
+		} else {
+			if strings.HasSuffix(path.Base(cgroupParent), ".slice") {
+				return nil, fmt.Errorf("cri-o configured with cgroupfs cgroup manager, but received systemd slice as parent: %s", cgroupParent)
+			}
+			cgPath := filepath.Join(cgroupParent, scopePrefix+"-"+id)
+			g.SetLinuxCgroupsPath(cgPath)
+		}
+	}
+	g.AddAnnotation(annotations.CgroupParent, cgroupParent)
+
+	sb, err := sandbox.New(id, namespace, name, kubeName, logDir, labels, kubeAnnotations, processLabel, mountLabel, metadata, shmPath, cgroupParent, privileged, trusted, resolvPath, hostname, portMappings)
+	if err != nil {
+		return nil, err
+	}
+
+	s.addSandbox(sb)
+	defer func() {
+		if err != nil {
+			s.removeSandbox(id)
+		}
+	}()
+
+	if err = s.PodIDIndex().Add(id); err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			if err := s.PodIDIndex().Delete(id); err != nil {
+				logrus.Warnf("couldn't delete pod id %s from idIndex", id)
+			}
+		}
+	}()
+
+	for k, v := range kubeAnnotations {
+		g.AddAnnotation(k, v)
+	}
+	for k, v := range labels {
+		g.AddAnnotation(k, v)
+	}
+
+	// extract linux sysctls from annotations and pass down to oci runtime
+	for key, value := range req.GetConfig().GetLinux().GetSysctls() {
+		g.AddLinuxSysctl(key, value)
+	}
+
+	// Set OOM score adjust of the infra container to be very low
+	// so it doesn't get killed.
+	g.SetProcessOOMScoreAdj(PodInfraOOMAdj)
+
+	g.SetLinuxResourcesCPUShares(PodInfraCPUshares)
+
+	// set up namespaces
+	if hostNetwork {
+		err = g.RemoveLinuxNamespace(string(runtimespec.NetworkNamespace))
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		// Create the sandbox network namespace
+		if err = sb.NetNsCreate(); err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			if err == nil {
+				return
+			}
+
+			if netnsErr := sb.NetNsRemove(); netnsErr != nil {
+				logrus.Warnf("Failed to remove networking namespace: %v", netnsErr)
+			}
+		}()
+
+		// Pass the created namespace path to the runtime
+		err = g.AddOrReplaceLinuxNamespace(string(runtimespec.NetworkNamespace), sb.NetNsPath())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if securityContext.GetNamespaceOptions().GetPid() == pb.NamespaceMode_NODE {
+		err = g.RemoveLinuxNamespace(string(runtimespec.PIDNamespace))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if securityContext.GetNamespaceOptions().GetIpc() == pb.NamespaceMode_NODE {
+		err = g.RemoveLinuxNamespace(string(runtimespec.IPCNamespace))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if !s.seccompEnabled {
+		g.Spec().Linux.Seccomp = nil
+	}
+
+	saveOptions := generate.ExportOptions{}
+	mountPoint, err := s.StorageRuntimeServer().StartContainer(id)
+	if err != nil {
+		return nil, fmt.Errorf("failed to mount container %s in pod sandbox %s(%s): %v", containerName, sb.Name(), id, err)
+	}
+	g.AddAnnotation(annotations.MountPoint, mountPoint)
+	g.SetRootPath(mountPoint)
+
+	hostnamePath := fmt.Sprintf("%s/hostname", podContainer.RunDir)
+	if err := ioutil.WriteFile(hostnamePath, []byte(hostname+"\n"), 0644); err != nil {
+		return nil, err
+	}
+	if err := label.Relabel(hostnamePath, mountLabel, true); err != nil && err != unix.ENOTSUP {
+		return nil, err
+	}
+	mnt := runtimespec.Mount{
+		Type:        "bind",
+		Source:      hostnamePath,
+		Destination: "/etc/hostname",
+		Options:     []string{"ro", "bind"},
+	}
+	g.AddMount(mnt)
+	g.AddAnnotation(annotations.HostnamePath, hostnamePath)
+	sb.AddHostnamePath(hostnamePath)
+
+	container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs().Path(), labels, g.Spec().Annotations, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
+	if err != nil {
+		return nil, err
+	}
+	container.SetSpec(g.Spec())
+	container.SetMountPoint(mountPoint)
+
+	sb.SetInfraContainer(container)
+
+	var ip string
+	ip, err = s.networkStart(hostNetwork, sb)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			s.networkStop(hostNetwork, sb)
+		}
+	}()
+
+	g.AddAnnotation(annotations.IP, ip)
+	sb.AddIP(ip)
+	sb.SetNamespaceOptions(securityContext.GetNamespaceOptions())
+
+	spp := req.GetConfig().GetLinux().GetSecurityContext().GetSeccompProfilePath()
+	g.AddAnnotation(annotations.SeccompProfilePath, spp)
+	sb.SetSeccompProfilePath(spp)
+	if !privileged {
+		if err = s.setupSeccomp(&g, spp); err != nil {
+			return nil, err
+		}
+	}
+
+	err = g.SaveToFile(filepath.Join(podContainer.Dir, "config.json"), saveOptions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.Name(), id, err)
+	}
+	if err = g.SaveToFile(filepath.Join(podContainer.RunDir, "config.json"), saveOptions); err != nil {
+		return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.Name(), id, err)
+	}
+
+	if err = s.runContainer(container, sb.CgroupParent()); err != nil {
+		return nil, err
+	}
+
+	s.addInfraContainer(container)
+
+	s.ContainerStateToDisk(container)
+
+	resp = &pb.RunPodSandboxResponse{PodSandboxId: id}
+	logrus.Debugf("RunPodSandboxResponse: %+v", resp)
+	return resp, nil
+}
+
+func setupShm(podSandboxRunDir, mountLabel string) (shmPath string, err error) {
+	shmPath = filepath.Join(podSandboxRunDir, "shm")
+	if err = os.Mkdir(shmPath, 0700); err != nil {
+		return "", err
+	}
+	shmOptions := "mode=1777,size=" + strconv.Itoa(sandbox.DefaultShmSize)
+	if err = unix.Mount("shm", shmPath, "tmpfs", unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV,
+		label.FormatMountLabel(shmOptions, mountLabel)); err != nil {
+		return "", fmt.Errorf("failed to mount shm tmpfs for pod: %v", err)
+	}
+	return shmPath, nil
+}

server/sandbox_run_unsupported.go

@@ -0,0 +1,14 @@
+// +build !linux
+
+package server
+
+import (
+	"fmt"
+
+	"golang.org/x/net/context"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+)
+
+func (s *Server) runPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest) (resp *pb.RunPodSandboxResponse, err error) {
+	return nil, fmt.Errorf("unsupported")
+}

server/sandbox_status.go

@@ -1,12 +1,14 @@
 package server
 
 import (
+	"encoding/json"
 	"time"
 
 	"github.com/kubernetes-incubator/cri-o/oci"
+	"github.com/kubernetes-incubator/cri-o/version"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // PodSandboxStatus returns the Status of the PodSandbox.
@@ -31,6 +33,16 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
 		rStatus = pb.PodSandboxState_SANDBOX_READY
 	}
 
+	linux := &pb.LinuxPodSandboxStatus{
+		Namespaces: &pb.Namespace{
+			Options: &pb.NamespaceOption{
+				Network: sb.NamespaceOptions().GetNetwork(),
+				Ipc:     sb.NamespaceOptions().GetIpc(),
+				Pid:     sb.NamespaceOptions().GetPid(),
+			},
+		},
+	}
+
 	sandboxID := sb.ID()
 	resp = &pb.PodSandboxStatusResponse{
 		Status: &pb.PodSandboxStatus{
@@ -41,9 +53,29 @@ func (s *Server) PodSandboxStatus(ctx context.Context, req *pb.PodSandboxStatusR
 			Labels:      sb.Labels(),
 			Annotations: sb.Annotations(),
 			Metadata:    sb.Metadata(),
+			Linux:       linux,
 		},
 	}
 
+	if req.Verbose {
+		resp = amendVerboseInfo(resp)
+	}
+
 	logrus.Debugf("PodSandboxStatusResponse: %+v", resp)
 	return resp, nil
 }
+
+// VersionPayload is a helper struct to create the JSON payload to show the version
+type VersionPayload struct {
+	Version string `json:"version"`
+}
+
+func amendVerboseInfo(resp *pb.PodSandboxStatusResponse) *pb.PodSandboxStatusResponse {
+	resp.Info = make(map[string]string)
+	bs, err := json.Marshal(VersionPayload{Version: version.Version})
+	if err != nil {
+		return resp // Just ignore the error and don't marshal the info
+	}
+	resp.Info["version"] = string(bs)
+	return resp
+}

server/sandbox_status_test.go

@@ -0,0 +1,119 @@
+package server
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/kubernetes-incubator/cri-o/lib"
+	"github.com/kubernetes-incubator/cri-o/lib/sandbox"
+	"github.com/kubernetes-incubator/cri-o/oci"
+	"github.com/kubernetes-incubator/cri-o/version"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+)
+
+func init() {
+	rand.Seed(time.Now().UTC().UnixNano())
+}
+
+func newTestContainerServerOrFailNow(t *testing.T) (cs *lib.ContainerServer, dirsToCleanUp []string) {
+	tmpdir := os.Getenv("TMPDIR")
+
+	config := lib.DefaultConfig()
+	runRoot, err := ioutil.TempDir(tmpdir, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	config.RootConfig.RunRoot = runRoot
+	root, err := ioutil.TempDir(tmpdir, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	config.RootConfig.Root = root
+	config.RootConfig.Storage = "vfs"
+	cs, err = lib.New(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cs, []string{runRoot, root}
+}
+
+func newTestSandboxOrFailNow(t *testing.T) (string, *sandbox.Sandbox) {
+	id := fmt.Sprintf("id-for-sandbox-%d", rand.Int())
+
+	sb, err := sandbox.New(id, "", "", "", "", nil, nil, "", "", nil, "", "", false, false, "", "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return id, sb
+}
+
+func newTestContainerOrFailNow(t *testing.T) *oci.Container {
+	id := fmt.Sprintf("id-for-container-%d", rand.Int())
+
+	c, err := oci.NewContainer(id, "", "", "", nil, nil, nil, nil, "", "", "", nil, "", false, false, false, false, false, "", time.Now(), "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return c
+}
+
+func setupServer(t *testing.T) (*Server, string, func()) {
+	containerServer, fs := newTestContainerServerOrFailNow(t)
+	teardown := func() {
+		for _, f := range fs {
+			defer os.RemoveAll(f)
+		}
+	}
+
+	server := &Server{ContainerServer: containerServer}
+	sandboxID, sb := newTestSandboxOrFailNow(t)
+	sb.SetInfraContainer(newTestContainerOrFailNow(t))
+	server.PodIDIndex().Add(sandboxID)
+	server.ContainerServer.AddSandbox(sb)
+
+	return server, sandboxID, teardown
+}
+
+func TestPodSandboxStatus(t *testing.T) {
+	server, sandboxID, teardown := setupServer(t)
+	defer teardown()
+
+	t.Run("Without verbose information", func(t *testing.T) {
+		resp, err := server.PodSandboxStatus(nil, &pb.PodSandboxStatusRequest{
+			PodSandboxId: sandboxID,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if resp.Status == nil {
+			t.Error("expected non nil resp.Status")
+		}
+		if resp.Info != nil {
+			t.Error("expected nil resp.Info")
+		}
+	})
+
+	t.Run("With verbose information", func(t *testing.T) {
+		resp, err := server.PodSandboxStatus(nil, &pb.PodSandboxStatusRequest{
+			PodSandboxId: sandboxID,
+			Verbose:      true,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		marshaledVersion := resp.Info["version"]
+		var versionPayload VersionPayload
+		must(t, json.Unmarshal([]byte(marshaledVersion), &versionPayload))
+
+		if version.Version != versionPayload.Version {
+			t.Errorf("expected: %s\ngot: %s", version.Version, versionPayload.Version)
+		}
+	})
+}

server/sandbox_stop.go

@@ -1,110 +1,16 @@
 package server
 
 import (
-	"fmt"
-	"time"
-
-	"github.com/containers/storage"
-	"github.com/docker/docker/pkg/mount"
-	"github.com/docker/docker/pkg/symlink"
-	"github.com/kubernetes-incubator/cri-o/lib/sandbox"
-	"github.com/kubernetes-incubator/cri-o/oci"
-	"github.com/opencontainers/selinux/go-selinux/label"
-	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
-	"golang.org/x/sys/unix"
-	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // StopPodSandbox stops the sandbox. If there are any running containers in the
 // sandbox, they should be force terminated.
 func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxRequest) (resp *pb.StopPodSandboxResponse, err error) {
-	const operation = "stop_pod_sandbox"
-	defer func() {
-		recordOperation(operation, time.Now())
-		recordError(operation, err)
-	}()
-
-	logrus.Debugf("StopPodSandboxRequest %+v", req)
-	sb, err := s.getPodSandboxFromRequest(req.PodSandboxId)
-	if err != nil {
-		if err == sandbox.ErrIDEmpty {
-			return nil, err
-		}
-
-		// If the sandbox isn't found we just return an empty response to adhere
-		// the the CRI interface which expects to not error out in not found
-		// cases.
-
-		resp = &pb.StopPodSandboxResponse{}
-		logrus.Warnf("could not get sandbox %s, it's probably been stopped already: %v", req.PodSandboxId, err)
-		logrus.Debugf("StopPodSandboxResponse %s: %+v", req.PodSandboxId, resp)
-		return resp, nil
-	}
-
-	if sb.Stopped() {
-		resp = &pb.StopPodSandboxResponse{}
-		logrus.Debugf("StopPodSandboxResponse %s: %+v", sb.ID(), resp)
-		return resp, nil
-	}
-
-	// Clean up sandbox networking and close its network namespace.
-	hostNetwork := sb.NetNsPath() == ""
-	s.networkStop(hostNetwork, sb)
-	if err := sb.NetNsRemove(); err != nil {
-		return nil, err
-	}
-
-	podInfraContainer := sb.InfraContainer()
-	containers := sb.Containers().List()
-	containers = append(containers, podInfraContainer)
-
-	for _, c := range containers {
-		cStatus := s.Runtime().ContainerStatus(c)
-		if cStatus.Status != oci.ContainerStateStopped {
-			if err := s.Runtime().StopContainer(ctx, c, 10); err != nil {
-				return nil, fmt.Errorf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
-			}
-			if c.ID() == podInfraContainer.ID() {
-				continue
-			}
-			if err := s.StorageRuntimeServer().StopContainer(c.ID()); err != nil && errors.Cause(err) != storage.ErrContainerUnknown {
-				// assume container already umounted
-				logrus.Warnf("failed to stop container %s in pod sandbox %s: %v", c.Name(), sb.ID(), err)
-			}
-		}
-		s.ContainerStateToDisk(c)
-	}
-
-	if err := label.ReleaseLabel(sb.ProcessLabel()); err != nil {
-		return nil, err
-	}
-
-	// unmount the shm for the pod
-	if sb.ShmPath() != "/dev/shm" {
-		// we got namespaces in the form of
-		// /var/run/containers/storage/overlay-containers/CID/userdata/shm
-		// but /var/run on most system is symlinked to /run so we first resolve
-		// the symlink and then try and see if it's mounted
-		fp, err := symlink.FollowSymlinkInScope(sb.ShmPath(), "/")
-		if err != nil {
-			return nil, err
-		}
-		if mounted, err := mount.Mounted(fp); err == nil && mounted {
-			if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil {
-				return nil, err
-			}
-		}
-	}
-	if err := s.StorageRuntimeServer().StopContainer(sb.ID()); err != nil && errors.Cause(err) != storage.ErrContainerUnknown {
-		logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err)
-	}
-
-	sb.SetStopped()
-	resp = &pb.StopPodSandboxResponse{}
-	logrus.Debugf("StopPodSandboxResponse %s: %+v", sb.ID(), resp)
-	return resp, nil
+	// platform dependent call
+	return s.stopPodSandbox(ctx, req)
 }
 
 // StopAllPodSandboxes removes all pod sandboxes