Merge pull request #311 from mlaventure/handle-sigkilled-shim

Handle sigkilled shim

Makefile

@@ -91,7 +91,7 @@ test: validate install bundles-rootfs
 	go test -bench=. -v $(shell go list ./... | grep -v /vendor | grep -v /integration-test ) -runtime=$(RUNTIME)
 ifneq ($(wildcard /.dockerenv), )
 	cd integration-test ; \
-	go test -check.v -check.timeout=$(TEST_TIMEOUT) timeout=$(TEST_SUITE_TIMEOUT) $(TESTFLAGS) github.com/docker/containerd/integration-test
+go test -check.v -check.timeout=$(TEST_TIMEOUT) $(TESTFLAGS) timeout=$(TEST_SUITE_TIMEOUT) github.com/docker/containerd/integration-test
 endif
 
 bench: shim validate install bundles-rootfs

containerd/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/docker/containerd/api/grpc/server"
 	"github.com/docker/containerd/api/grpc/types"
 	"github.com/docker/containerd/api/http/pprof"
+	"github.com/docker/containerd/osutils"
 	"github.com/docker/containerd/supervisor"
 	"github.com/docker/docker/pkg/listeners"
 	"github.com/rcrowley/go-metrics"
@@ -157,6 +158,7 @@ func main() {
 func daemon(context *cli.Context) error {
 	s := make(chan os.Signal, 2048)
 	signal.Notify(s, syscall.SIGTERM, syscall.SIGINT)
+	osutils.SetSubreaper(1)
 	sv, err := supervisor.New(
 		context.String("state-dir"),
 		context.String("runtime"),

integration-test/start_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"syscall"
 	"time"
@@ -488,3 +489,50 @@ func swapEnabled() bool {
 	_, err := os.Stat("/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes")
 	return err == nil
 }
+
+func (cs *ContainerdSuite) TestSigkillShimReuseName(t *check.C) {
+	bundleName := "busybox-top"
+	if err := CreateBusyboxBundle(bundleName, []string{"top"}); err != nil {
+		t.Fatal(err)
+	}
+	containerID := "top"
+	c, err := cs.StartContainer(containerID, bundleName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Sigkill the shim
+	exec.Command("pkill", "-9", "containerd-shim").Run()
+
+	// Wait for it to be reaped
+	for _, evt := range []types.Event{
+		{
+			Type:   "start-container",
+			Id:     containerID,
+			Status: 0,
+			Pid:    "",
+		},
+		{
+			Type:   "exit",
+			Id:     containerID,
+			Status: 128 + 9,
+			Pid:    "init",
+		},
+	} {
+		ch := c.GetEventsChannel()
+		select {
+		case e := <-ch:
+			evt.Timestamp = e.Timestamp
+
+			t.Assert(*e, checker.Equals, evt)
+		case <-time.After(2 * time.Second):
+			t.Fatal("Container took more than 2 seconds to terminate")
+		}
+	}
+
+	// Start a new continer with the same name
+	c, err = cs.StartContainer(containerID, bundleName)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

runtime/container.go

@@ -253,6 +253,8 @@ func (c *container) Delete() error {
 	args = append(args, "delete", c.id)
 	if b, derr := exec.Command(c.runtime, args...).CombinedOutput(); err != nil {
 		err = fmt.Errorf("%s: %q", derr, string(b))
+	} else if len(b) > 0 {
+		logrus.Debugf("%v %v: %q", c.runtime, args, string(b))
 	}
 	return err
 }

runtime/process.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 	"syscall"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/docker/containerd/specs"
 	"golang.org/x/sys/unix"
 )
@@ -193,8 +194,58 @@ func (p *process) Resize(w, h int) error {
 	return err
 }
 
-func (p *process) ExitStatus() (int, error) {
+func (p *process) handleSigkilledShim(rst int, rerr error) (int, error) {
+	if rerr == nil || p.cmd == nil || p.cmd.Process == nil {
+		return rst, rerr
+	}
+
+	// Possible that the shim was SIGKILLED
+	e := unix.Kill(p.cmd.Process.Pid, 0)
+	if e != syscall.ESRCH {
+		return rst, rerr
+	}
+
+	// Ensure we got the shim ProcessState
+	<-p.cmdDoneCh
+
+	shimStatus := p.cmd.ProcessState.Sys().(syscall.WaitStatus)
+	if shimStatus.Signaled() && shimStatus.Signal() == syscall.SIGKILL {
+		logrus.Debugf("containerd: ExitStatus(container: %s, process: %s): shim was SIGKILL'ed reaping its child with pid %d", p.container.id, p.id, p.pid)
+
+		var (
+			status unix.WaitStatus
+			rusage unix.Rusage
+			wpid   int
+		)
+
+		for wpid == 0 {
+			wpid, e = unix.Wait4(p.pid, &status, unix.WNOHANG, &rusage)
+			if e != nil {
+				logrus.Debugf("containerd: ExitStatus(container: %s, process: %s): Wait4(%d): %v", p.container.id, p.id, p.pid, rerr)
+				return rst, rerr
+			}
+		}
+
+		if wpid == p.pid {
+			rerr = nil
+			rst = 128 + int(shimStatus.Signal())
+		} else {
+			logrus.Errorf("containerd: ExitStatus(container: %s, process: %s): unexpected returned pid from wait4 %v (expected %v)", p.container.id, p.id, wpid, p.pid)
+		}
+
+		p.stateLock.Lock()
+		p.state = Stopped
+		p.stateLock.Unlock()
+	}
+
+	return rst, rerr
+}
+
+func (p *process) ExitStatus() (rst int, rerr error) {
 	data, err := ioutil.ReadFile(filepath.Join(p.root, ExitStatusFile))
+	defer func() {
+		rst, rerr = p.handleSigkilledShim(rst, rerr)
+	}()
 	if err != nil {
 		if os.IsNotExist(err) {
 			return -1, ErrProcessNotExited