Add shim for reattach of processes

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Remove runtime files from containerd

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Update supervisor for orphaned containers

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Remove ctr/container.go back to rpc calls

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add attach to loaded container

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add monitor based on epoll for process exits

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Convert pids in containerd to string

This is so that we no longer care about linux or system level pids and
processes in containerd have user defined process id(pid) kinda like the
exec process ids that docker has today.

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add reaper back to containerd

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Implement list containers with new process model

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Implement restore of processes

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add NONBLOCK to exit fifo open

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Implement tty reattach

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Fix race in exit pipe creation

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add delete to shim

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Update shim to use pid-file and not stdout

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

.gitignore

@@ -1,4 +1,5 @@
 containerd/containerd
+containerd-shim/containerd-shim
 bin/
 ctr/ctr
 hack/benchmark

Makefile

@@ -1,4 +1,4 @@
-BUILDTAGS=libcontainer
+BUILDTAGS=
 
 # if this session isn't interactive, then we don't want to allocate a
 # TTY, which would fail, but if it is interactive, we do want to attach
@@ -13,7 +13,7 @@ DOCKER_RUN := docker run --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
 
 export GOPATH:=$(CURDIR)/vendor:$(GOPATH)
 
-all: client daemon
+all: client daemon shim
 
 bin:
 	mkdir -p bin/
@@ -27,6 +27,9 @@ client: bin
 daemon: bin
 	cd containerd && go build -tags "$(BUILDTAGS)" -o ../bin/containerd
 
+shim: bin
+	cd containerd-shim && go build -tags "$(BUILDTAGS)" -o ../bin/containerd-shim
+
 dbuild:
 	@docker build --rm --force-rm -t "$(DOCKER_IMAGE)" .
 

api/grpc/server/server.go

@@ -33,10 +33,6 @@ func (s *apiServer) CreateContainer(ctx context.Context, c *types.CreateContaine
 	e := supervisor.NewEvent(supervisor.StartContainerEventType)
 	e.ID = c.Id
 	e.BundlePath = c.BundlePath
-	e.Stdout = c.Stdout
-	e.Stderr = c.Stderr
-	e.Stdin = c.Stdin
-	e.Console = c.Console
 	e.StartResponse = make(chan supervisor.StartResponse, 1)
 	if c.Checkpoint != "" {
 		e.Checkpoint = &runtime.Checkpoint{
@@ -49,14 +45,16 @@ func (s *apiServer) CreateContainer(ctx context.Context, c *types.CreateContaine
 	}
 	sr := <-e.StartResponse
 	return &types.CreateContainerResponse{
-		Pid: uint32(sr.Pid),
+		Stdin:  sr.Stdin,
+		Stdout: sr.Stdout,
+		Stderr: sr.Stderr,
 	}, nil
 }
 
 func (s *apiServer) Signal(ctx context.Context, r *types.SignalRequest) (*types.SignalResponse, error) {
 	e := supervisor.NewEvent(supervisor.SignalEventType)
 	e.ID = r.Id
-	e.Pid = int(r.Pid)
+	e.Pid = r.Pid
 	e.Signal = syscall.Signal(int(r.Signal))
 	s.sv.SendEvent(e)
 	if err := <-e.Err; err != nil {
@@ -79,7 +77,7 @@ func (s *apiServer) AddProcess(ctx context.Context, r *types.AddProcessRequest)
 	}
 	e := supervisor.NewEvent(supervisor.AddProcessEventType)
 	e.ID = r.Id
-	e.Process = process
+	e.ProcessSpec = process
 	e.Console = r.Console
 	e.Stdin = r.Stdin
 	e.Stdout = r.Stdout
@@ -88,7 +86,7 @@ func (s *apiServer) AddProcess(ctx context.Context, r *types.AddProcessRequest)
 	if err := <-e.Err; err != nil {
 		return nil, err
 	}
-	return &types.AddProcessResponse{Pid: uint32(e.Pid)}, nil
+	return &types.AddProcessResponse{}, nil
 }
 
 func (s *apiServer) CreateCheckpoint(ctx context.Context, r *types.CreateCheckpointRequest) (*types.CreateCheckpointResponse, error) {
@@ -140,11 +138,12 @@ func (s *apiServer) ListCheckpoint(ctx context.Context, r *types.ListCheckpointR
 	if container == nil {
 		return nil, grpc.Errorf(codes.NotFound, "no such containers")
 	}
+	var out []*types.Checkpoint
+	/*
 		checkpoints, err := container.Checkpoints()
 		if err != nil {
 			return nil, err
 		}
-	var out []*types.Checkpoint
 		for _, c := range checkpoints {
 			out = append(out, &types.Checkpoint{
 				Name:        c.Name,
@@ -155,6 +154,7 @@ func (s *apiServer) ListCheckpoint(ctx context.Context, r *types.ListCheckpointR
 				//Timestamp:   c.Timestamp,
 			})
 		}
+	*/
 	return &types.ListCheckpointResponse{Checkpoints: out}, nil
 }
 
@@ -167,7 +167,6 @@ func (s *apiServer) State(ctx context.Context, r *types.StateRequest) (*types.St
 	m := s.sv.Machine()
 	state := &types.StateResponse{
 		Machine: &types.Machine{
-			Id:     m.ID,
 			Cpus:   uint32(m.Cpus),
 			Memory: uint64(m.Cpus),
 		},
@@ -179,13 +178,9 @@ func (s *apiServer) State(ctx context.Context, r *types.StateRequest) (*types.St
 		}
 		var procs []*types.Process
 		for _, p := range processes {
-			pid, err := p.Pid()
-			if err != nil {
-				logrus.WithField("error", err).Error("get process pid")
-			}
 			oldProc := p.Spec()
 			procs = append(procs, &types.Process{
-				Pid:      uint32(pid),
+				Pid:      p.ID(),
 				Terminal: oldProc.Terminal,
 				Args:     oldProc.Args,
 				Env:      oldProc.Env,
@@ -231,7 +226,7 @@ func (s *apiServer) Events(r *types.EventsRequest, stream types.API_EventsServer
 			ev = &types.Event{
 				Type:   "exit",
 				Id:     evt.ID,
-				Pid:    uint32(evt.Pid),
+				Pid:    evt.Pid,
 				Status: uint32(evt.Status),
 			}
 		case supervisor.OOMEventType:

api/grpc/types/api.pb.go

@@ -65,11 +65,7 @@ var _ = math.Inf
 type CreateContainerRequest struct {
 	Id         string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
 	BundlePath string `protobuf:"bytes,2,opt,name=bundlePath" json:"bundlePath,omitempty"`
-	Stdin      string `protobuf:"bytes,3,opt,name=stdin" json:"stdin,omitempty"`
+	Checkpoint string `protobuf:"bytes,3,opt,name=checkpoint" json:"checkpoint,omitempty"`
-	Stdout     string `protobuf:"bytes,4,opt,name=stdout" json:"stdout,omitempty"`
-	Stderr     string `protobuf:"bytes,5,opt,name=stderr" json:"stderr,omitempty"`
-	Console    string `protobuf:"bytes,6,opt,name=console" json:"console,omitempty"`
-	Checkpoint string `protobuf:"bytes,7,opt,name=checkpoint" json:"checkpoint,omitempty"`
 }
 
 func (m *CreateContainerRequest) Reset()                    { *m = CreateContainerRequest{} }
@@ -78,7 +74,9 @@ func (*CreateContainerRequest) ProtoMessage()               {}
 func (*CreateContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
 
 type CreateContainerResponse struct {
-	Pid uint32 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"`
+	Stdin  string `protobuf:"bytes,1,opt,name=stdin" json:"stdin,omitempty"`
+	Stdout string `protobuf:"bytes,2,opt,name=stdout" json:"stdout,omitempty"`
+	Stderr string `protobuf:"bytes,3,opt,name=stderr" json:"stderr,omitempty"`
 }
 
 func (m *CreateContainerResponse) Reset()                    { *m = CreateContainerResponse{} }
@@ -88,7 +86,7 @@ func (*CreateContainerResponse) Descriptor() ([]byte, []int) { return fileDescri
 
 type SignalRequest struct {
 	Id     string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
-	Pid    uint32 `protobuf:"varint,2,opt,name=pid" json:"pid,omitempty"`
+	Pid    string `protobuf:"bytes,2,opt,name=pid" json:"pid,omitempty"`
 	Signal uint32 `protobuf:"varint,3,opt,name=signal" json:"signal,omitempty"`
 }
 
@@ -142,7 +140,6 @@ func (*User) ProtoMessage()               {}
 func (*User) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
 
 type AddProcessResponse struct {
-	Pid uint32 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"`
 }
 
 func (m *AddProcessResponse) Reset()                    { *m = AddProcessResponse{} }
@@ -249,7 +246,7 @@ func (*ContainerState) ProtoMessage()               {}
 func (*ContainerState) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
 
 type Process struct {
-	Pid      uint32   `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"`
+	Pid      string   `protobuf:"bytes,1,opt,name=pid" json:"pid,omitempty"`
 	Terminal bool     `protobuf:"varint,2,opt,name=terminal" json:"terminal,omitempty"`
 	User     *User    `protobuf:"bytes,3,opt,name=user" json:"user,omitempty"`
 	Args     []string `protobuf:"bytes,4,rep,name=args" json:"args,omitempty"`
@@ -291,9 +288,8 @@ func (m *Container) GetProcesses() []*Process {
 
 // Machine is information about machine on which containerd is run
 type Machine struct {
-	Id     string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+	Cpus   uint32 `protobuf:"varint,1,opt,name=cpus" json:"cpus,omitempty"`
-	Cpus   uint32 `protobuf:"varint,2,opt,name=cpus" json:"cpus,omitempty"`
+	Memory uint64 `protobuf:"varint,2,opt,name=memory" json:"memory,omitempty"`
-	Memory uint64 `protobuf:"varint,3,opt,name=memory" json:"memory,omitempty"`
 }
 
 func (m *Machine) Reset()                    { *m = Machine{} }
@@ -358,7 +354,7 @@ type Event struct {
 	Id         string       `protobuf:"bytes,2,opt,name=id" json:"id,omitempty"`
 	Status     uint32       `protobuf:"varint,3,opt,name=status" json:"status,omitempty"`
 	BundlePath string       `protobuf:"bytes,4,opt,name=bundlePath" json:"bundlePath,omitempty"`
-	Pid        uint32       `protobuf:"varint,5,opt,name=pid" json:"pid,omitempty"`
+	Pid        string       `protobuf:"bytes,5,opt,name=pid" json:"pid,omitempty"`
 	Signal     uint32       `protobuf:"varint,7,opt,name=signal" json:"signal,omitempty"`
 	Process    *Process     `protobuf:"bytes,8,opt,name=process" json:"process,omitempty"`
 	Containers []*Container `protobuf:"bytes,9,rep,name=containers" json:"containers,omitempty"`
@@ -1087,96 +1083,95 @@ var _API_serviceDesc = grpc.ServiceDesc{
 }
 
 var fileDescriptor0 = []byte{
-	// 1454 bytes of a gzipped FileDescriptorProto
+	// 1440 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x6e, 0x1c, 0x45,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x72, 0xdc, 0x44,
-	0x17, 0xf6, 0xcc, 0xf4, 0x6c, 0x67, 0x16, 0xdb, 0xed, 0xd8, 0x19, 0xcf, 0xff, 0x87, 0x24, 0x4d,
+	0x17, 0xf6, 0xec, 0x33, 0x67, 0x16, 0xdb, 0xf2, 0x92, 0xf1, 0xfc, 0x7f, 0x88, 0x11, 0x81, 0xa4,
-	0x08, 0x11, 0x8a, 0xac, 0xe0, 0xb0, 0x84, 0x70, 0x01, 0xc1, 0x89, 0x12, 0x50, 0x02, 0x56, 0x62,
+	0xa8, 0x94, 0x2b, 0x38, 0x2c, 0x21, 0x5c, 0x40, 0x70, 0x52, 0x09, 0x54, 0x02, 0xae, 0xc4, 0xa6,
-	0x23, 0x71, 0xc3, 0xa8, 0xdd, 0x5d, 0x8c, 0x8b, 0xe9, 0x8d, 0xee, 0x6a, 0x2f, 0xaf, 0x80, 0x78,
+	0x8a, 0x1b, 0xa6, 0x64, 0xa9, 0x19, 0x8b, 0xd1, 0x48, 0x42, 0x6a, 0x79, 0x79, 0x05, 0x9e, 0x87,
-	0x1c, 0xc4, 0x03, 0x20, 0x71, 0xcf, 0x73, 0xf0, 0x14, 0x9c, 0x5a, 0xba, 0x7a, 0x99, 0xc5, 0x70,
+	0xe2, 0x01, 0xa8, 0xe2, 0x9e, 0xe7, 0xe0, 0x29, 0x38, 0xdd, 0x7d, 0xd4, 0x5a, 0x66, 0x09, 0x5c,
-	0xc1, 0xcd, 0x48, 0x55, 0x75, 0xce, 0x77, 0xce, 0xf9, 0xce, 0xd2, 0x55, 0x03, 0x5d, 0x3b, 0xa2,
+	0x70, 0x33, 0x55, 0xdd, 0x7d, 0xce, 0x77, 0xbe, 0xb3, 0xaa, 0x7b, 0xa0, 0x63, 0x85, 0xee, 0x41,
-	0x7b, 0x51, 0x1c, 0xb2, 0xd0, 0x6c, 0xb2, 0xcb, 0x88, 0x24, 0xd6, 0x2f, 0x35, 0xd8, 0x39, 0x88,
+	0x18, 0x05, 0x3c, 0x30, 0x1a, 0xfc, 0x3a, 0x64, 0xb1, 0xf9, 0x02, 0x76, 0x8f, 0x22, 0x66, 0x71,
-	0x89, 0xcd, 0xc8, 0x41, 0x18, 0x30, 0x9b, 0x06, 0x24, 0x7e, 0x4d, 0x7e, 0x4a, 0x49, 0xc2, 0x4c,
+	0x76, 0x14, 0xf8, 0xdc, 0x72, 0x7d, 0x16, 0xbd, 0x62, 0x3f, 0x27, 0x2c, 0xe6, 0x06, 0x40, 0xd5,
-	0x80, 0x3a, 0x75, 0x47, 0xb5, 0x5b, 0xb5, 0x7b, 0x5d, 0x13, 0x17, 0x27, 0x69, 0xe0, 0x7a, 0xe4,
+	0x75, 0x86, 0x95, 0xfd, 0xca, 0xdd, 0x8e, 0x81, 0x8b, 0xb3, 0xc4, 0x77, 0x3c, 0x76, 0x6c, 0xf1,
-	0xd0, 0x66, 0xa7, 0xa3, 0xba, 0xd8, 0x1b, 0x40, 0x33, 0x61, 0x2e, 0x0d, 0x46, 0x0d, 0xb1, 0x1c,
+	0xf3, 0x61, 0x35, 0xdd, 0xb3, 0xcf, 0x99, 0x3d, 0x0d, 0x03, 0xd7, 0xe7, 0xc3, 0x9a, 0xd8, 0x33,
-	0x42, 0x0b, 0x97, 0x61, 0xca, 0x46, 0x46, 0x61, 0x4d, 0xe2, 0x78, 0xd4, 0x14, 0xeb, 0x75, 0x68,
+	0x9f, 0xc3, 0x8d, 0x39, 0xb4, 0x38, 0x0c, 0xfc, 0x98, 0x19, 0x7d, 0x68, 0xc4, 0xdc, 0x71, 0x7d,
-	0x3b, 0x61, 0x90, 0x84, 0x1e, 0x19, 0xb5, 0x32, 0x4c, 0xe7, 0x94, 0x38, 0xb3, 0x28, 0xa4, 0x01,
+	0x42, 0x1c, 0x40, 0x13, 0x97, 0x41, 0xc2, 0x09, 0x4d, 0xad, 0x59, 0x14, 0x11, 0xd2, 0x43, 0xe8,
-	0x1b, 0xb5, 0xf9, 0x9e, 0x75, 0x17, 0xae, 0xcf, 0x79, 0x93, 0x44, 0xa8, 0x46, 0xcc, 0x1e, 0x34,
+	0xbf, 0x76, 0x27, 0xbe, 0xe5, 0x2d, 0xa2, 0xd3, 0x85, 0x5a, 0x88, 0x8b, 0x4c, 0x53, 0x4a, 0x4a,
-	0x22, 0xe5, 0xcf, 0xc0, 0x7a, 0x04, 0x83, 0x37, 0x74, 0x1a, 0xd8, 0xde, 0x22, 0x67, 0x95, 0x24,
+	0xcd, 0xbe, 0xb9, 0x01, 0x83, 0x54, 0x53, 0x99, 0x36, 0x7f, 0xad, 0xc0, 0xe6, 0x63, 0xc7, 0x39,
-	0xf7, 0x72, 0x20, 0xdc, 0x10, 0x92, 0xc2, 0xcd, 0x81, 0xb5, 0x01, 0xc3, 0x4c, 0x53, 0x02, 0x5b,
+	0x8e, 0x02, 0x9b, 0xc5, 0xf1, 0x22, 0xc0, 0x0d, 0x68, 0x73, 0x16, 0xcd, 0x5c, 0x81, 0x22, 0x50,
-	0xbf, 0xd6, 0x60, 0xf3, 0x89, 0xeb, 0x1e, 0xc6, 0xa1, 0x43, 0x92, 0x64, 0x11, 0xe0, 0x06, 0x74,
+	0xdb, 0xc6, 0x1e, 0xd4, 0x93, 0x98, 0x29, 0x36, 0xdd, 0xc3, 0xee, 0x81, 0x8c, 0xd6, 0xc1, 0x29,
-	0x18, 0x89, 0x7d, 0xca, 0x51, 0x38, 0x6a, 0xc7, 0xdc, 0x05, 0x23, 0x4d, 0x48, 0x2c, 0x30, 0x7b,
+	0x6e, 0x19, 0x3d, 0xa8, 0x5b, 0xd1, 0x24, 0x1e, 0xd6, 0xf7, 0x6b, 0x8a, 0x0b, 0xf3, 0x2f, 0x86,
-	0xfb, 0xbd, 0x3d, 0x41, 0xe6, 0xde, 0x31, 0x6e, 0x99, 0x7d, 0x30, 0xec, 0x78, 0x9a, 0x20, 0x0b,
+	0x8d, 0x74, 0x61, 0x5f, 0x3a, 0xc3, 0xa6, 0x04, 0xd5, 0x1e, 0xb7, 0x4a, 0x1e, 0xb7, 0x4b, 0x1e,
-	0x0d, 0xe9, 0x0b, 0x09, 0xce, 0x90, 0x02, 0xb5, 0x70, 0xce, 0x5d, 0x15, 0xbe, 0xa6, 0xaf, 0x5d,
+	0x77, 0xe4, 0x7a, 0x1d, 0x5a, 0x36, 0xd2, 0x0d, 0x3c, 0x36, 0x04, 0x0a, 0x41, 0x5d, 0xda, 0x43,
-	0xa1, 0xaf, 0x53, 0xa1, 0xaf, 0x5b, 0xa5, 0x0f, 0x04, 0x55, 0x8f, 0xc0, 0x10, 0xf6, 0x10, 0x34,
+	0xd0, 0x84, 0x98, 0xf6, 0xc5, 0x62, 0x42, 0xae, 0xf7, 0x8d, 0x5d, 0x18, 0x58, 0x8e, 0xe3, 0x72,
-	0xcd, 0x78, 0xe1, 0x8b, 0xa9, 0x0e, 0x7d, 0x07, 0x86, 0xb6, 0xeb, 0x52, 0x46, 0x43, 0x74, 0xfc,
+	0x37, 0x40, 0xe2, 0xcf, 0x5c, 0x27, 0x46, 0xba, 0x35, 0x0c, 0xc1, 0x36, 0x18, 0x79, 0x7f, 0x29,
-	0x39, 0x75, 0x13, 0x74, 0xb7, 0x81, 0x14, 0xdc, 0x06, 0xb3, 0x18, 0xef, 0x22, 0x7e, 0x5f, 0xea,
+	0x0c, 0x2f, 0x74, 0x72, 0x74, 0xda, 0x16, 0xc5, 0xe2, 0xdd, 0x42, 0x5e, 0xab, 0xd2, 0xff, 0x4d,
-	0x3c, 0xe8, 0x0c, 0x2d, 0x22, 0xe6, 0x9d, 0x52, 0x0a, 0xeb, 0x82, 0x8c, 0x4d, 0x45, 0x46, 0xae,
+	0xf2, 0x3f, 0xd3, 0x34, 0x47, 0x30, 0x9c, 0x47, 0x23, 0x4b, 0x0f, 0xe0, 0xc6, 0x13, 0xe6, 0xb1,
-	0x69, 0x8d, 0x61, 0x34, 0x8f, 0xa6, 0xd8, 0x7f, 0x08, 0xd7, 0x9f, 0x12, 0x8f, 0x5c, 0x65, 0x09,
+	0x37, 0x59, 0xc2, 0x40, 0xfa, 0xd6, 0x8c, 0xa9, 0x3c, 0x0a, 0xc0, 0x79, 0x25, 0x02, 0x7c, 0x07,
-	0x59, 0x0d, 0x6c, 0x9f, 0xc8, 0xd2, 0xe3, 0x80, 0xf3, 0x4a, 0x0a, 0xf0, 0x6d, 0xd8, 0x7e, 0x49,
+	0x76, 0x5e, 0xb8, 0x31, 0x5f, 0x09, 0x67, 0x7e, 0x0f, 0x90, 0x09, 0x68, 0x70, 0x6d, 0x8a, 0x5d,
-	0x13, 0xb6, 0x12, 0xce, 0xfa, 0x0e, 0x20, 0x17, 0xd0, 0xe0, 0xda, 0x14, 0xb9, 0xa0, 0x4c, 0x65,
+	0xb9, 0x9c, 0x92, 0x8b, 0x41, 0xe4, 0x76, 0x28, 0x73, 0xdb, 0x36, 0xb6, 0xa0, 0x9b, 0xf8, 0xee,
-	0x1a, 0x69, 0x61, 0x4e, 0x24, 0x12, 0xdd, 0x31, 0xb7, 0xa0, 0x97, 0x06, 0xf4, 0xe2, 0x4d, 0xe8,
+	0xd5, 0xeb, 0xc0, 0x9e, 0x32, 0x2e, 0xb2, 0x2a, 0x36, 0x45, 0xee, 0xce, 0x99, 0xe7, 0x61, 0x5e,
-	0xcc, 0x08, 0x4b, 0x44, 0xa1, 0x77, 0x44, 0x22, 0x4f, 0x89, 0xe7, 0x89, 0x3a, 0xef, 0x58, 0x9f,
+	0x71, 0x69, 0x7e, 0x01, 0xbb, 0x65, 0xfb, 0x54, 0xd6, 0xef, 0x41, 0x37, 0x8b, 0x56, 0x8c, 0xd6,
-	0xc3, 0x4e, 0xd5, 0xbe, 0x62, 0xf8, 0x2e, 0xf4, 0x72, 0xb6, 0x12, 0xb4, 0xd6, 0x58, 0x4c, 0xd7,
+	0x6a, 0x8b, 0xc3, 0x35, 0x80, 0xde, 0x6b, 0x8e, 0xd1, 0x22, 0xe2, 0xe6, 0x3e, 0x0c, 0x74, 0x8f,
-	0x10, 0xfa, 0x6f, 0x18, 0xb2, 0xa5, 0x1c, 0xb7, 0x6e, 0xc1, 0x50, 0xb7, 0x83, 0x38, 0x90, 0xc5,
+	0xc8, 0x03, 0x55, 0x0f, 0x16, 0x4f, 0x62, 0x72, 0x67, 0x0a, 0x2d, 0xca, 0x60, 0x5a, 0xef, 0xff,
-	0x61, 0xb3, 0x34, 0x51, 0xe1, 0xcc, 0xa0, 0xad, 0xd2, 0x59, 0x4a, 0xe3, 0x7f, 0x52, 0xb8, 0x96,
+	0x5d, 0xad, 0x9a, 0x1e, 0x74, 0x34, 0x9d, 0xe5, 0x39, 0x2a, 0xcd, 0x01, 0xd9, 0xa9, 0xc6, 0xdb,
-	0x07, 0x5d, 0xed, 0xce, 0xf2, 0x1c, 0x55, 0x46, 0x86, 0x9c, 0x11, 0xb7, 0xa1, 0x1b, 0x49, 0x3f,
+	0xd0, 0x09, 0x15, 0x4f, 0xa6, 0xec, 0x74, 0x0f, 0x07, 0x44, 0x21, 0xe5, 0x9f, 0xb9, 0xd6, 0x90,
-	0x89, 0xb4, 0xd3, 0xdb, 0x1f, 0x2a, 0x17, 0x32, 0xff, 0xf3, 0xd0, 0xc4, 0xd8, 0xc0, 0xfa, 0x68,
+	0xd6, 0xee, 0x40, 0xeb, 0xa5, 0x65, 0x9f, 0xa3, 0x31, 0x81, 0x6f, 0x87, 0xe4, 0x73, 0x5f, 0x08,
-	0xbf, 0xb2, 0x9d, 0x53, 0x34, 0x56, 0xb5, 0xe5, 0x44, 0x28, 0xa4, 0x9b, 0xdc, 0x27, 0x7e, 0x18,
+	0xce, 0xd8, 0x2c, 0x88, 0xae, 0xa5, 0xbd, 0xba, 0xf9, 0x1d, 0x4e, 0x01, 0x15, 0x35, 0x0a, 0xf7,
-	0x5f, 0x0a, 0x3b, 0x86, 0xf5, 0x2d, 0x8e, 0x07, 0xc9, 0xa0, 0xa2, 0xfe, 0x0e, 0x16, 0x6a, 0xe6,
+	0x6d, 0x2c, 0xce, 0x94, 0x67, 0x1a, 0xed, 0x8d, 0x34, 0xda, 0xda, 0x81, 0x5b, 0xd0, 0x9a, 0x29,
-	0x73, 0xc6, 0xfc, 0x46, 0xc6, 0xbc, 0x0e, 0xe6, 0x26, 0xb4, 0x7d, 0x69, 0x4b, 0xd5, 0x72, 0xe6,
+	0x7c, 0xaa, 0xdf, 0x94, 0x10, 0x59, 0x35, 0x9f, 0xc0, 0xee, 0x69, 0xe8, 0xbc, 0x69, 0xea, 0x65,
-	0x9c, 0xf2, 0xc0, 0x7a, 0x0a, 0x3b, 0xc7, 0x91, 0x7b, 0xd5, 0xb0, 0xcc, 0x47, 0x4e, 0x3e, 0x82,
+	0x93, 0xa5, 0x9a, 0xb2, 0x23, 0x37, 0xd4, 0x8c, 0xda, 0x83, 0x1b, 0x73, 0x28, 0x54, 0xb0, 0xeb,
-	0x64, 0x48, 0x82, 0x05, 0x6b, 0x17, 0xae, 0xcf, 0xa1, 0xa8, 0xe2, 0x5d, 0x87, 0xc1, 0xb3, 0x33,
+	0xd0, 0x7f, 0x7a, 0xc1, 0xb0, 0x22, 0xd2, 0x7c, 0xff, 0x59, 0x81, 0x86, 0xdc, 0x11, 0x1e, 0x0b,
-	0x82, 0xd5, 0x91, 0xe5, 0xfe, 0xcf, 0x1a, 0x34, 0xc5, 0x0e, 0x8f, 0x98, 0x3b, 0xa3, 0x6c, 0x48,
+	0x32, 0x64, 0x43, 0xd9, 0xcb, 0xcd, 0x40, 0x8d, 0xdf, 0x2f, 0x45, 0xbb, 0x9e, 0x1f, 0x7d, 0x8d,
-	0x7b, 0xf5, 0x7c, 0x54, 0x68, 0xfc, 0x41, 0x85, 0x79, 0xa3, 0x38, 0x13, 0x9b, 0x95, 0x99, 0xd8,
+	0x12, 0xc1, 0x96, 0x54, 0x40, 0xbf, 0x29, 0x15, 0x72, 0xc6, 0xcc, 0x27, 0xa2, 0x18, 0xbe, 0xce,
-	0x16, 0x6b, 0x8c, 0x5b, 0xa5, 0x45, 0x0c, 0x9f, 0xf9, 0xa4, 0x94, 0xe9, 0xeb, 0x2e, 0xa1, 0xaf,
+	0x92, 0xf0, 0x15, 0x27, 0x00, 0x2c, 0x9b, 0x00, 0xbf, 0x55, 0xa0, 0xf7, 0x0d, 0xe3, 0x97, 0x41,
-	0x3c, 0x0d, 0x60, 0xd9, 0x34, 0xf8, 0xad, 0x06, 0xfd, 0xaf, 0x09, 0x3b, 0x0f, 0xe3, 0x19, 0x4f,
+	0x34, 0x15, 0x49, 0x8a, 0x4b, 0x2d, 0x87, 0x75, 0x1a, 0x5d, 0x8d, 0xcf, 0xae, 0x39, 0x96, 0x85,
-	0x52, 0x52, 0x69, 0x3f, 0xac, 0xd9, 0xf8, 0x62, 0x72, 0x72, 0xc9, 0x88, 0xcc, 0xae, 0xc1, 0xe3,
+	0xcc, 0xa6, 0xf0, 0x07, 0x77, 0x8e, 0x2d, 0xd5, 0x68, 0x35, 0xb9, 0xb7, 0x09, 0x9d, 0x57, 0x57,
-	0xc1, 0x9d, 0x43, 0x5b, 0x36, 0x9d, 0xc8, 0xb0, 0xb9, 0x09, 0xdd, 0xd7, 0x17, 0x13, 0x1c, 0x8f,
+	0x63, 0x9c, 0x82, 0x41, 0xa4, 0x7a, 0x4f, 0x8a, 0xe1, 0x96, 0x13, 0x05, 0x61, 0xc8, 0x94, 0xa7,
-	0x61, 0x2c, 0xfb, 0x50, 0x88, 0xe1, 0x96, 0x1b, 0x87, 0x51, 0x44, 0x64, 0xa4, 0x06, 0x07, 0x3b,
+	0x75, 0x01, 0x76, 0x92, 0x82, 0x35, 0x53, 0x29, 0xdc, 0x09, 0x09, 0xac, 0x95, 0x82, 0x9d, 0x68,
-	0xca, 0xc0, 0x5a, 0x99, 0x14, 0xee, 0x44, 0x0a, 0xac, 0x9d, 0x81, 0x1d, 0x69, 0xb0, 0x4e, 0x41,
+	0xb0, 0x76, 0x4e, 0x2c, 0x05, 0xeb, 0xc8, 0xaa, 0x9a, 0x41, 0xfb, 0x28, 0x4c, 0x4e, 0x63, 0x6b,
-	0x2c, 0x03, 0xeb, 0x8a, 0xaa, 0xf2, 0xa1, 0x73, 0x10, 0xa5, 0xc7, 0x89, 0x3d, 0x25, 0x7c, 0x12,
+	0xc2, 0x44, 0xf7, 0xf3, 0x80, 0x5b, 0xde, 0x38, 0x11, 0x4b, 0x49, 0xbd, 0x6e, 0x6c, 0x43, 0x2f,
-	0xb0, 0x90, 0xd9, 0xde, 0x24, 0xe5, 0x4b, 0xe1, 0xba, 0x61, 0x5e, 0x83, 0x7e, 0x44, 0x62, 0xac,
+	0x64, 0x11, 0xd6, 0x25, 0xed, 0x56, 0x31, 0x50, 0x75, 0xe3, 0x7f, 0xb0, 0x25, 0x97, 0x63, 0xd7,
-	0x4b, 0xb5, 0x5b, 0x47, 0xa2, 0x0c, 0xf3, 0x7f, 0xb0, 0x25, 0x96, 0x13, 0x1a, 0x4c, 0x66, 0x24,
+	0x1f, 0x4f, 0x59, 0xe4, 0x33, 0x6f, 0x16, 0x38, 0x8c, 0xfc, 0xd8, 0x83, 0x4d, 0x7d, 0x28, 0x9a,
-	0x0e, 0x88, 0xe7, 0x87, 0x2e, 0x51, 0x71, 0xec, 0xc2, 0xa6, 0x3e, 0xe4, 0x8d, 0x29, 0x8e, 0x44,
+	0x51, 0x1e, 0x49, 0x7f, 0xcc, 0x13, 0x18, 0x9c, 0x9c, 0xe3, 0x37, 0x97, 0x7b, 0xae, 0x3f, 0x79,
-	0x3c, 0xd6, 0x11, 0x0c, 0x8f, 0x4e, 0xf1, 0x5b, 0xcd, 0x3c, 0x1a, 0x4c, 0x9f, 0xda, 0xcc, 0xe6,
+	0x62, 0x71, 0x4b, 0x8c, 0x7a, 0xc4, 0x77, 0x03, 0x27, 0x26, 0x83, 0xa8, 0xcd, 0x95, 0x08, 0x73,
-	0xdf, 0x00, 0xc4, 0xa7, 0xa1, 0x9b, 0x28, 0x83, 0xa8, 0xcd, 0xa4, 0x08, 0x71, 0x27, 0xd9, 0x91,
+	0xc6, 0xe9, 0x91, 0x0a, 0x1a, 0xce, 0xf8, 0xec, 0x88, 0xbb, 0x33, 0x32, 0x68, 0xfe, 0x20, 0x9d,
-	0x24, 0x0d, 0x87, 0x7f, 0x7e, 0xc4, 0xa8, 0xaf, 0x0c, 0x5a, 0xdf, 0x8b, 0x20, 0x24, 0xf1, 0x16,
+	0x50, 0x81, 0x37, 0xa1, 0x93, 0x91, 0xad, 0xc8, 0x7c, 0xad, 0xa7, 0xf9, 0x4a, 0x1d, 0x3d, 0x80,
-	0x74, 0x73, 0x67, 0x6b, 0x22, 0x5f, 0xeb, 0x59, 0xbe, 0xb2, 0x40, 0xf7, 0x60, 0x9d, 0x69, 0x2f,
+	0x75, 0xae, 0x59, 0x8c, 0xb1, 0x6a, 0x2d, 0xea, 0x8d, 0x1d, 0x92, 0x2c, 0x72, 0x34, 0x3f, 0x07,
-	0x26, 0x58, 0xb5, 0xb6, 0xea, 0x8d, 0x6d, 0x25, 0x59, 0xf6, 0xd1, 0xfa, 0x0c, 0xe0, 0x95, 0x68,
+	0x78, 0x29, 0x5b, 0x51, 0x32, 0xc6, 0x79, 0x98, 0x0f, 0x10, 0x06, 0x7a, 0x66, 0x5d, 0xe9, 0xe8,
-	0x45, 0xe1, 0x31, 0xce, 0xc6, 0x22, 0x41, 0x48, 0xb4, 0x6f, 0x5f, 0x68, 0x76, 0xf8, 0x16, 0xc6,
+	0x88, 0x2d, 0xf4, 0xe9, 0x47, 0xcb, 0xf5, 0x6c, 0xba, 0x0b, 0xd4, 0xcd, 0xbf, 0x2a, 0xd0, 0x55,
-	0xf4, 0x83, 0x4d, 0x3d, 0x07, 0x2b, 0x46, 0x3a, 0xf8, 0x57, 0x0d, 0x7a, 0x12, 0x41, 0x3a, 0x89,
+	0x08, 0x8a, 0x24, 0x42, 0xd8, 0xd8, 0x7e, 0x29, 0xc4, 0x7e, 0x8a, 0x58, 0xfc, 0xc2, 0xe4, 0x6c,
-	0x10, 0x0e, 0xb6, 0x5f, 0x06, 0x71, 0x2b, 0x43, 0x2c, 0x7f, 0x6d, 0x0a, 0x36, 0xb1, 0x0c, 0x93,
+	0x62, 0x19, 0xc6, 0x97, 0x56, 0x48, 0x56, 0x6a, 0xcb, 0xc4, 0xee, 0x40, 0x4f, 0x65, 0x83, 0x04,
-	0x73, 0x3b, 0x52, 0x56, 0x1a, 0xcb, 0xc4, 0xde, 0x85, 0xbe, 0xcc, 0x86, 0x12, 0x34, 0x96, 0x09,
+	0xeb, 0xcb, 0x04, 0xef, 0x89, 0xef, 0x31, 0x32, 0x91, 0xf3, 0xaf, 0x7b, 0x78, 0xb3, 0x20, 0x21,
-	0xde, 0xe7, 0x1f, 0x6a, 0xf4, 0x44, 0xcc, 0xc2, 0xde, 0xfe, 0x8d, 0x92, 0x84, 0xf0, 0x71, 0x4f,
+	0x39, 0x1e, 0xc8, 0xdf, 0xa7, 0x3e, 0x8f, 0xae, 0x47, 0xf7, 0x00, 0xb2, 0x95, 0x68, 0xbb, 0x29,
-	0xfc, 0x3e, 0x0b, 0x58, 0x7c, 0x39, 0xbe, 0x0f, 0x90, 0xaf, 0x78, 0xdb, 0xcd, 0xc8, 0xa5, 0xaa,
+	0xbb, 0xa6, 0xca, 0x46, 0x4f, 0x2e, 0x2c, 0x2f, 0x21, 0xcf, 0x1f, 0x55, 0x1f, 0x56, 0xcc, 0xaf,
-	0x6c, 0x8c, 0xe4, 0xcc, 0xf6, 0x52, 0x15, 0xf9, 0xe3, 0xfa, 0xa3, 0x9a, 0xf5, 0x15, 0xac, 0x7f,
+	0x61, 0xfd, 0x4b, 0x6f, 0xea, 0x06, 0x39, 0x15, 0x94, 0x9a, 0x59, 0x3f, 0x05, 0x11, 0xf9, 0x2b,
-	0xe1, 0xcd, 0x68, 0x58, 0x50, 0x41, 0x29, 0xdf, 0xfe, 0x31, 0x8c, 0x55, 0xbc, 0x7c, 0x49, 0x03,
+	0x96, 0xae, 0x8f, 0x4b, 0x15, 0x2e, 0xec, 0xfb, 0x20, 0xa4, 0x09, 0xaa, 0xf1, 0x54, 0xbd, 0xfc,
-	0x5c, 0x4a, 0xba, 0xb0, 0xef, 0xc3, 0x48, 0x4d, 0x53, 0x8d, 0x27, 0xeb, 0xe5, 0xf7, 0x06, 0x40,
+	0x5e, 0x03, 0xc8, 0xc0, 0x8c, 0x47, 0x30, 0x72, 0x83, 0x31, 0x96, 0xd4, 0x85, 0x6b, 0x33, 0xd5,
-	0x0e, 0x66, 0x3e, 0x86, 0x31, 0x0d, 0x27, 0x58, 0x52, 0x67, 0xd4, 0x21, 0xb2, 0x05, 0x26, 0x31,
+	0x02, 0xe3, 0x88, 0xd9, 0x49, 0x14, 0xbb, 0x17, 0x8c, 0x46, 0xe0, 0x2e, 0xf9, 0x52, 0xe6, 0xf0,
-	0x71, 0xd2, 0x38, 0xa1, 0x67, 0x44, 0x8d, 0xc0, 0x1d, 0x15, 0x4b, 0xd5, 0x87, 0x0f, 0x61, 0x3b,
+	0x11, 0xec, 0x64, 0xba, 0x4e, 0x4e, 0xad, 0xba, 0x52, 0xed, 0x01, 0x6c, 0xa1, 0x1a, 0x0e, 0xae,
-	0xd7, 0x75, 0x0b, 0x6a, 0xf5, 0x95, 0x6a, 0x0f, 0x61, 0x0b, 0xd5, 0x70, 0x70, 0xa5, 0x25, 0xa5,
+	0xa4, 0xa0, 0x54, 0x5b, 0xa9, 0xf4, 0x29, 0xec, 0xe5, 0x78, 0x8a, 0x4a, 0xcd, 0xa9, 0xd6, 0x57,
-	0xc6, 0x4a, 0xa5, 0x4f, 0x60, 0xb7, 0xe0, 0x27, 0xaf, 0xd4, 0x82, 0xaa, 0xb1, 0x52, 0xf5, 0x23,
+	0xaa, 0x7e, 0x0c, 0xbb, 0xa8, 0x7a, 0x69, 0xb9, 0xbc, 0xac, 0xd7, 0xf8, 0x07, 0x3c, 0x67, 0x2c,
-	0xd8, 0x41, 0xd5, 0x73, 0x9b, 0xb2, 0xaa, 0x5e, 0xf3, 0x1f, 0xf8, 0xe9, 0x93, 0x78, 0x5a, 0xf2,
+	0x9a, 0x14, 0x78, 0x36, 0x57, 0x2a, 0x7d, 0x00, 0x9b, 0xa8, 0x54, 0xb2, 0xd3, 0x7a, 0x93, 0x4a,
-	0xb3, 0xb5, 0x52, 0xe9, 0x7d, 0xd8, 0x44, 0xa5, 0x8a, 0x9d, 0xf6, 0x55, 0x2a, 0x09, 0x71, 0x18,
+	0xcc, 0x6c, 0x8e, 0x53, 0x25, 0xa7, 0xd2, 0x5e, 0xa5, 0x62, 0x3e, 0x86, 0xde, 0xf3, 0x64, 0xc2,
-	0x4e, 0x95, 0x82, 0x4a, 0x67, 0x95, 0x8a, 0xf5, 0x04, 0xfa, 0x2f, 0xd2, 0x29, 0x61, 0xde, 0x89,
+	0xb8, 0x77, 0xa6, 0xab, 0xff, 0xdf, 0x36, 0xd0, 0x2f, 0x55, 0xe8, 0x1e, 0x4d, 0xa2, 0x20, 0x09,
-	0xae, 0xfe, 0x7f, 0xdb, 0x40, 0x3f, 0xd7, 0xa1, 0x77, 0x30, 0x8d, 0xc3, 0x34, 0x2a, 0x75, 0xb9,
+	0x0b, 0x5d, 0xae, 0x6a, 0x78, 0xae, 0xcb, 0x95, 0xcc, 0x5d, 0xe8, 0xa9, 0x0f, 0x28, 0x89, 0xa9,
-	0xac, 0xe1, 0xb9, 0x2e, 0x97, 0x32, 0xf7, 0xa0, 0x2f, 0x3f, 0xa0, 0x4a, 0x4c, 0x36, 0x97, 0x39,
+	0xe6, 0x32, 0xe6, 0x4b, 0x5d, 0x5c, 0x5c, 0xce, 0x04, 0x67, 0x12, 0x2c, 0xb6, 0x57, 0xae, 0xfc,
-	0x5f, 0xea, 0xfc, 0x12, 0x73, 0xc2, 0x7d, 0x56, 0x82, 0xe5, 0xf6, 0x2a, 0x94, 0xdf, 0xa7, 0x30,
+	0x3e, 0x83, 0xfe, 0xb9, 0x72, 0x84, 0x24, 0x55, 0x2a, 0x6f, 0xa7, 0x96, 0x33, 0x82, 0x07, 0x79,
-	0x38, 0x95, 0x81, 0x28, 0x49, 0x99, 0xca, 0x3b, 0x99, 0xe5, 0xdc, 0xc1, 0xbd, 0x62, 0xc0, 0xb2,
+	0x87, 0x55, 0x13, 0x3d, 0x87, 0xcd, 0xb9, 0xcd, 0x62, 0x2f, 0x99, 0xf9, 0x5e, 0xea, 0x1e, 0x6e,
-	0x89, 0x5e, 0xc0, 0xe6, 0xdc, 0x66, 0xb9, 0x97, 0xac, 0x62, 0x2f, 0xf5, 0xf6, 0xb7, 0x14, 0x6c,
+	0x11, 0x6c, 0x5e, 0x4b, 0x36, 0x58, 0x08, 0x0d, 0xc5, 0xe7, 0x7d, 0xe8, 0xfb, 0xea, 0xa3, 0xa3,
-	0x51, 0x4b, 0x34, 0x58, 0x04, 0x4d, 0xe9, 0xcf, 0x7b, 0x30, 0x08, 0xe4, 0x47, 0x47, 0x33, 0xd1,
+	0x23, 0x51, 0xcb, 0x29, 0x16, 0x3e, 0x48, 0x18, 0x0d, 0x5b, 0xf2, 0x5b, 0x18, 0x8d, 0x7c, 0x6c,
-	0x28, 0x28, 0x96, 0x3e, 0x48, 0xc8, 0x86, 0x23, 0xfc, 0x5b, 0xc8, 0x46, 0x91, 0x5b, 0xcc, 0x07,
+	0x31, 0x1f, 0xa2, 0x22, 0x50, 0x6c, 0x16, 0x52, 0xf8, 0x47, 0xea, 0xc6, 0xb6, 0xe8, 0xbd, 0x70,
-	0xaf, 0x08, 0x14, 0xf3, 0x23, 0x45, 0xff, 0x58, 0xde, 0xde, 0x16, 0x3d, 0x24, 0xf6, 0xff, 0x68,
+	0xf8, 0x47, 0x03, 0x6a, 0x8f, 0x8f, 0xbf, 0x32, 0x5e, 0xc1, 0x7a, 0xe9, 0xbd, 0x63, 0xa4, 0x63,
-	0x42, 0xe3, 0xc9, 0xe1, 0x97, 0xe6, 0x6b, 0x58, 0xaf, 0x3c, 0x73, 0xcc, 0x6c, 0xac, 0x2c, 0x7e,
+	0x65, 0xf1, 0xab, 0x6a, 0xf4, 0xd6, 0xb2, 0x63, 0xba, 0x38, 0xac, 0x09, 0xcc, 0xd2, 0xad, 0x42,
-	0x8c, 0x8d, 0xdf, 0x5a, 0x76, 0xac, 0x2e, 0x0e, 0x6b, 0x1c, 0xb3, 0x72, 0xab, 0xd0, 0x98, 0x8b,
+	0x63, 0x2e, 0xbe, 0xb3, 0x68, 0xcc, 0x65, 0x97, 0x91, 0x35, 0xe3, 0x13, 0x68, 0xaa, 0x37, 0x91,
-	0xef, 0x2c, 0x1a, 0x73, 0xd9, 0x65, 0x64, 0xcd, 0xfc, 0x18, 0x5a, 0xf2, 0xb1, 0x64, 0x5e, 0x53,
+	0xb1, 0x4d, 0xb2, 0x85, 0xc7, 0xd5, 0x68, 0xa7, 0xb4, 0xab, 0x15, 0x8f, 0x00, 0xb2, 0x97, 0x84,
-	0xb2, 0xa5, 0x57, 0xd7, 0x78, 0xbb, 0xb2, 0xab, 0x15, 0x0f, 0x00, 0xf2, 0x27, 0x86, 0x39, 0x52,
+	0x31, 0x24, 0xb1, 0xb9, 0xc7, 0xd4, 0x68, 0x6f, 0xc1, 0x89, 0x06, 0x39, 0x85, 0x8d, 0xf2, 0x53,
-	0x62, 0x73, 0xaf, 0xac, 0xf1, 0xee, 0x82, 0x13, 0x0d, 0x72, 0x0c, 0x1b, 0xd5, 0x67, 0x83, 0x59,
+	0xc1, 0x28, 0xc5, 0xa1, 0x7c, 0xb1, 0x1f, 0xdd, 0x5a, 0x7a, 0x9e, 0x87, 0x2d, 0x3f, 0x18, 0x34,
-	0xe1, 0xa1, 0x7a, 0xc9, 0x1f, 0xdf, 0x5c, 0x7a, 0x5e, 0x84, 0xad, 0x3e, 0x1e, 0x34, 0xec, 0x92,
+	0xec, 0x92, 0xe7, 0x87, 0x86, 0x5d, 0xfa, 0xd2, 0x58, 0x33, 0xbe, 0x85, 0x41, 0xf1, 0xae, 0x6f,
-	0xa7, 0x88, 0x86, 0x5d, 0xfa, 0xea, 0x58, 0x33, 0xbf, 0x81, 0x61, 0xf9, 0xde, 0x6f, 0xfe, 0x5f,
+	0xfc, 0x9f, 0x94, 0x16, 0x3e, 0x41, 0x46, 0x37, 0x97, 0x9c, 0x6a, 0xc0, 0x0f, 0x55, 0xe9, 0xe2,
-	0x29, 0x2d, 0x7c, 0x8e, 0x8c, 0x6f, 0x2c, 0x39, 0xd5, 0x80, 0x1f, 0xc8, 0xd2, 0xc5, 0xbb, 0x46,
+	0x5d, 0x23, 0x8d, 0x72, 0xee, 0x21, 0x30, 0xda, 0x2e, 0x6e, 0x6a, 0xad, 0xfb, 0xd0, 0x54, 0x37,
-	0xc6, 0x72, 0xe1, 0x51, 0x30, 0xbe, 0x56, 0xde, 0xd4, 0x5a, 0x0f, 0xa0, 0x25, 0x6f, 0x90, 0x3a,
+	0x48, 0x9d, 0xb2, 0xc2, 0x85, 0x72, 0xd4, 0xcb, 0xef, 0x9a, 0x6b, 0xf7, 0x2b, 0x38, 0xa5, 0xda,
-	0x65, 0xa5, 0x0b, 0xe5, 0xb8, 0x5f, 0xdc, 0xb5, 0xd6, 0x1e, 0xd4, 0x70, 0x4a, 0x75, 0x9e, 0x13,
+	0xcf, 0x18, 0x57, 0xf5, 0x9c, 0x37, 0x35, 0xa7, 0x22, 0x37, 0x85, 0xca, 0x59, 0x53, 0xfe, 0x17,
-	0x26, 0xeb, 0xb9, 0x68, 0x6a, 0x4e, 0x45, 0x6c, 0x72, 0x95, 0x93, 0x96, 0xf8, 0x0f, 0xe1, 0xe1,
+	0xf0, 0xe0, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x15, 0x52, 0xba, 0x18, 0x10, 0x00, 0x00,
-	0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa6, 0xf1, 0xd0, 0x3c, 0x50, 0x10, 0x00, 0x00,
 }

api/grpc/types/api.proto

@@ -18,20 +18,19 @@ service API {
 message CreateContainerRequest {
 	string id = 1; // ID of container
 	string bundlePath = 2; // path to OCI bundle
-	string stdin = 3; // path to the file where stdin will be read (optional)
-	string stdout = 4; // path to file where stdout will be written (optional)
-	string stderr = 5; // path to file where stderr will be written (optional)
-	string console = 6; // path to the console for a container (optional)
 	string checkpoint = 7; // checkpoint name if you want to create immediate checkpoint (optional)
 }
 
 message CreateContainerResponse {
 	uint32 pid = 1; // PID of the containers main process
+	string stdin = 1; // path to the file where stdin will be read (optional)
+	string stdout = 2; // path to file where stdout will be written (optional)
+	string stderr = 3; // path to file where stderr will be written (optional)
 }
 
 message SignalRequest {
 	string id = 1; // ID of container
-	uint32 pid = 2; // PID of process inside container
+	string pid = 2; // PID of process inside container
 	uint32 signal = 3; // Signal which will be sent, you can find value in "man 7 signal"
 }
 
@@ -58,7 +57,6 @@ message User {
 }
 
 message AddProcessResponse {
-	uint32 pid = 1; // PID of process is returned in case of success
 }
 
 message CreateCheckpointRequest {
@@ -101,7 +99,7 @@ message ContainerState {
 }
 
 message Process {
-	uint32 pid = 1;
+	string pid = 1;
 	bool terminal = 2; // Use tty for container stdio
 	User user = 3; // User under which process will be run
 	repeated string args = 4; // Arguments for process, first is binary path itself
@@ -119,9 +117,8 @@ message Container {
 
 // Machine is information about machine on which containerd is run
 message Machine {
-	string id = 1; // ID of machine
+	uint32 cpus = 1; // number of cpus
-	uint32 cpus = 2; // number of cpus
+	uint64 memory = 2; // amount of memory
-	uint64 memory = 3; // amount of memory
 }
 
 // StateResponse is information about containerd daemon
@@ -147,7 +144,7 @@ message Event {
 	string id = 2;
 	uint32 status = 3;
 	string bundlePath = 4;
-	uint32 pid = 5;
+	string pid = 5;
 	uint32 signal = 7;
 	Process process = 8;
 	repeated Container containers = 9;

containerd-shim/main.go

@@ -0,0 +1,210 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/containerd/util"
+	"github.com/opencontainers/runc/libcontainer"
+	"github.com/opencontainers/specs"
+)
+
+const (
+	bufferSize = 2048
+)
+
+type stdio struct {
+	stdin   *os.File
+	stdout  *os.File
+	stderr  *os.File
+	console string
+}
+
+func (s *stdio) Close() error {
+	err := s.stdin.Close()
+	if oerr := s.stdout.Close(); err == nil {
+		err = oerr
+	}
+	if oerr := s.stderr.Close(); err == nil {
+		err = oerr
+	}
+	return err
+}
+
+// containerd-shim is a small shim that sits in front of a runc implementation
+// that allows it to be repartented to init and handle reattach from the caller.
+//
+// the cwd of the shim should be the bundle for the container.  Arg1 should be the path
+// to the state directory where the shim can locate fifos and other information.
+//
+//   └── shim
+//    ├── control
+//    ├── stderr
+//    ├── stdin
+//    ├── stdout
+//    ├── pid
+//    └── exit
+func main() {
+	if len(os.Args) < 2 {
+		logrus.Fatal("shim: no arguments provided")
+	}
+	// start handling signals as soon as possible so that things are properly reaped
+	// or if runc exits before we hit the handler
+	signals := make(chan os.Signal, bufferSize)
+	signal.Notify(signals)
+	// set the shim as the subreaper for all orphaned processes created by the container
+	if err := util.SetSubreaper(1); err != nil {
+		logrus.WithField("error", err).Fatal("shim: set as subreaper")
+	}
+	// open the exit pipe
+	f, err := os.OpenFile(filepath.Join(os.Args[1], "exit"), syscall.O_WRONLY, 0)
+	if err != nil {
+		logrus.WithField("error", err).Fatal("shim: open exit pipe")
+	}
+	defer f.Close()
+	// open the fifos for use with the command
+	std, err := openContainerSTDIO(os.Args[1])
+	if err != nil {
+		logrus.WithField("error", err).Fatal("shim: open container STDIO from fifo")
+	}
+	// star the container process by invoking runc
+	runcPid, err := startRunc(std, os.Args[2])
+	if err != nil {
+		logrus.WithField("error", err).Fatal("shim: start runc")
+	}
+	var exitShim bool
+	for s := range signals {
+		logrus.WithField("signal", s).Debug("shim: received signal")
+		switch s {
+		case syscall.SIGCHLD:
+			exits, err := util.Reap()
+			if err != nil {
+				logrus.WithField("error", err).Error("shim: reaping child processes")
+			}
+			for _, e := range exits {
+				// check to see if runc is one of the processes that has exited
+				if e.Pid == runcPid {
+					exitShim = true
+					logrus.WithFields(logrus.Fields{"pid": e.Pid, "status": e.Status}).Info("shim: runc exited")
+
+					if err := writeInt(filepath.Join(os.Args[1], "exitStatus"), e.Status); err != nil {
+						logrus.WithFields(logrus.Fields{"error": err, "status": e.Status}).Error("shim: write exit status")
+					}
+				}
+			}
+		}
+		// runc has exited so the shim can also exit
+		if exitShim {
+			if err := std.Close(); err != nil {
+				logrus.WithField("error", err).Error("shim: close stdio")
+			}
+			if err := deleteContainer(os.Args[2]); err != nil {
+				logrus.WithField("error", err).Error("shim: delete runc state")
+			}
+			return
+		}
+	}
+}
+
+// startRunc starts runc detached and returns the container's pid
+func startRunc(s *stdio, id string) (int, error) {
+	pidFile := filepath.Join(os.Args[1], "pid")
+	cmd := exec.Command("runc", "--id", id, "start", "-d", "--console", s.console, "--pid-file", pidFile)
+	cmd.Stdin = s.stdin
+	cmd.Stdout = s.stdout
+	cmd.Stderr = s.stderr
+	// set the parent death signal to SIGKILL so that if the shim dies the container
+	// process also dies
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Pdeathsig: syscall.SIGKILL,
+	}
+	if err := cmd.Run(); err != nil {
+		return -1, err
+	}
+	data, err := ioutil.ReadFile(pidFile)
+	if err != nil {
+		return -1, err
+	}
+	return strconv.Atoi(string(data))
+}
+
+func deleteContainer(id string) error {
+	return exec.Command("runc", "--id", id, "delete").Run()
+}
+
+// openContainerSTDIO opens the pre-created fifo's for use with the container
+// in RDWR so that they remain open if the other side stops listening
+func openContainerSTDIO(dir string) (*stdio, error) {
+	s := &stdio{}
+	spec, err := getSpec()
+	if err != nil {
+		return nil, err
+	}
+	if spec.Process.Terminal {
+		console, err := libcontainer.NewConsole(int(spec.Process.User.UID), int(spec.Process.User.GID))
+		if err != nil {
+			return nil, err
+		}
+		s.console = console.Path()
+		stdin, err := os.OpenFile(filepath.Join(dir, "stdin"), syscall.O_RDWR, 0)
+		if err != nil {
+			return nil, err
+		}
+		go func() {
+			io.Copy(console, stdin)
+		}()
+		stdout, err := os.OpenFile(filepath.Join(dir, "stdout"), syscall.O_RDWR, 0)
+		if err != nil {
+			return nil, err
+		}
+		go func() {
+			io.Copy(stdout, console)
+			console.Close()
+		}()
+		return s, nil
+	}
+	for name, dest := range map[string]**os.File{
+		"stdin":  &s.stdin,
+		"stdout": &s.stdout,
+		"stderr": &s.stderr,
+	} {
+		f, err := os.OpenFile(filepath.Join(dir, name), syscall.O_RDWR, 0)
+		if err != nil {
+			return nil, err
+		}
+		*dest = f
+	}
+	return s, nil
+}
+
+func getSpec() (*specs.Spec, error) {
+	var s specs.Spec
+	f, err := os.Open("config.json")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	if err := json.NewDecoder(f).Decode(&s); err != nil {
+		return nil, err
+	}
+	return &s, nil
+}
+
+func writeInt(path string, i int) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = fmt.Fprintf(f, "%d", i)
+	return err
+}

containerd/main.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"os/signal"
 	"runtime"
 	"sync"
 	"syscall"
@@ -29,11 +30,6 @@ const (
 )
 
 var daemonFlags = []cli.Flag{
-	cli.StringFlag{
-		Name:  "id",
-		Value: getDefaultID(),
-		Usage: "unique containerd id to identify the instance",
-	},
 	cli.BoolFlag{
 		Name:  "debug",
 		Usage: "enable debug output in the logs",
@@ -88,7 +84,6 @@ func main() {
 	}
 	app.Action = func(context *cli.Context) {
 		if err := daemon(
-			context.String("id"),
 			context.String("listen"),
 			context.String("state-dir"),
 			context.Int("concurrency"),
@@ -172,9 +167,12 @@ func processMetrics() {
 	}()
 }
 
-func daemon(id, address, stateDir string, concurrency int, oom bool) error {
+func daemon(address, stateDir string, concurrency int, oom bool) error {
+	// setup a standard reaper so that we don't leave any zombies if we are still alive
+	// this is just good practice because we are spawning new processes
+	go reapProcesses()
 	tasks := make(chan *supervisor.StartTask, concurrency*100)
-	sv, err := supervisor.New(id, stateDir, tasks, oom)
+	sv, err := supervisor.New(stateDir, tasks, oom)
 	if err != nil {
 		return err
 	}
@@ -184,17 +182,6 @@ func daemon(id, address, stateDir string, concurrency int, oom bool) error {
 		w := supervisor.NewWorker(sv, wg)
 		go w.Start()
 	}
-	// only set containerd as the subreaper if it is not an init process
-	if pid := os.Getpid(); pid != 1 {
-		logrus.WithFields(logrus.Fields{
-			"pid": pid,
-		}).Debug("containerd is not init, set as subreaper")
-		if err := setSubReaper(); err != nil {
-			return err
-		}
-	}
-	// start the signal handler in the background.
-	go startSignalHandler(sv)
 	if err := sv.Start(); err != nil {
 		return err
 	}
@@ -211,6 +198,19 @@ func daemon(id, address, stateDir string, concurrency int, oom bool) error {
 	return s.Serve(l)
 }
 
+func reapProcesses() {
+	s := make(chan os.Signal, 2048)
+	signal.Notify(s, syscall.SIGCHLD)
+	if err := util.SetSubreaper(1); err != nil {
+		logrus.WithField("error", err).Error("containerd: set subpreaper")
+	}
+	for range s {
+		if _, err := util.Reap(); err != nil {
+			logrus.WithField("error", err).Error("containerd: reap child processes")
+		}
+	}
+}
+
 // getDefaultID returns the hostname for the instance host
 func getDefaultID() string {
 	hostname, err := os.Hostname()

containerd/reap_linux.go

@@ -1,67 +0,0 @@
-// +build linux
-
-package main
-
-import (
-	"os"
-	"os/signal"
-	"syscall"
-
-	"github.com/Sirupsen/logrus"
-	"github.com/docker/containerd/supervisor"
-	"github.com/docker/containerd/util"
-	"github.com/opencontainers/runc/libcontainer/utils"
-)
-
-const signalBufferSize = 2048
-
-func startSignalHandler(supervisor *supervisor.Supervisor) {
-	logrus.WithFields(logrus.Fields{
-		"bufferSize": signalBufferSize,
-	}).Debug("containerd: starting signal handler")
-	signals := make(chan os.Signal, signalBufferSize)
-	signal.Notify(signals)
-	for s := range signals {
-		switch s {
-		case syscall.SIGTERM, syscall.SIGINT:
-			supervisor.Stop(signals)
-		case syscall.SIGCHLD:
-			exits, err := reap()
-			if err != nil {
-				logrus.WithField("error", err).Error("containerd: reaping child processes")
-			}
-			for _, e := range exits {
-				supervisor.SendEvent(e)
-			}
-		}
-	}
-	supervisor.Close()
-	os.Exit(0)
-}
-
-func reap() (exits []*supervisor.Event, err error) {
-	var (
-		ws  syscall.WaitStatus
-		rus syscall.Rusage
-	)
-	for {
-		pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, &rus)
-		if err != nil {
-			if err == syscall.ECHILD {
-				return exits, nil
-			}
-			return exits, err
-		}
-		if pid <= 0 {
-			return exits, nil
-		}
-		e := supervisor.NewEvent(supervisor.ExitEventType)
-		e.Pid = pid
-		e.Status = utils.ExitStatus(ws)
-		exits = append(exits, e)
-	}
-}
-
-func setSubReaper() error {
-	return util.SetSubreaper(1)
-}

ctr/container.go

@@ -4,10 +4,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"text/tabwriter"
 	"time"
@@ -15,7 +15,6 @@ import (
 	"github.com/codegangsta/cli"
 	"github.com/docker/containerd/api/grpc/types"
 	"github.com/docker/docker/pkg/term"
-	"github.com/opencontainers/runc/libcontainer"
 	"github.com/opencontainers/specs"
 	netcontext "golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -45,6 +44,7 @@ var containersCommand = cli.Command{
 		listCommand,
 		startCommand,
 		statsCommand,
+		attachCommand,
 	},
 	Action: listContainers,
 }
@@ -62,15 +62,97 @@ func listContainers(context *cli.Context) {
 		fatal(err.Error(), 1)
 	}
 	w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
-	fmt.Fprint(w, "ID\tPATH\tSTATUS\tPID1\n")
+	fmt.Fprint(w, "ID\tPATH\tSTATUS\tPROCESSES\n")
 	for _, c := range resp.Containers {
-		fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", c.Id, c.BundlePath, c.Status, c.Processes[0].Pid)
+		procs := []string{}
+		for _, p := range c.Processes {
+			procs = append(procs, p.Pid)
+		}
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", c.Id, c.BundlePath, c.Status, strings.Join(procs, ","))
 	}
 	if err := w.Flush(); err != nil {
 		fatal(err.Error(), 1)
 	}
 }
 
+var attachCommand = cli.Command{
+	Name:  "attach",
+	Usage: "attach to a running container",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "state-dir",
+			Value: "/run/containerd",
+			Usage: "runtime state directory",
+		},
+		cli.StringFlag{
+			Name:  "pid,p",
+			Value: "init",
+			Usage: "specify the process id to attach to",
+		},
+	},
+	Action: func(context *cli.Context) {
+		var (
+			id  = context.Args().First()
+			pid = context.String("pid")
+		)
+		if id == "" {
+			fatal("container id cannot be empty", 1)
+		}
+		c := getClient(context)
+		events, err := c.Events(netcontext.Background(), &types.EventsRequest{})
+		if err != nil {
+			fatal(err.Error(), 1)
+		}
+		type bundleState struct {
+			Bundle string `json:"bundle"`
+		}
+		f, err := os.Open(filepath.Join(context.String("state-dir"), id, "state.json"))
+		if err != nil {
+			fatal(err.Error(), 1)
+		}
+		var s bundleState
+		err = json.NewDecoder(f).Decode(&s)
+		f.Close()
+		if err != nil {
+			fatal(err.Error(), 1)
+		}
+		mkterm, err := readTermSetting(s.Bundle)
+		if err != nil {
+			fatal(err.Error(), 1)
+		}
+		if mkterm {
+			s, err := term.SetRawTerminal(os.Stdin.Fd())
+			if err != nil {
+				fatal(err.Error(), 1)
+			}
+			state = s
+		}
+		if err := attachStdio(
+			filepath.Join(context.String("state-dir"), id, pid, "stdin"),
+			filepath.Join(context.String("state-dir"), id, pid, "stdout"),
+			filepath.Join(context.String("state-dir"), id, pid, "stderr"),
+		); err != nil {
+			fatal(err.Error(), 1)
+		}
+		go func() {
+			io.Copy(stdin, os.Stdin)
+			if state != nil {
+				term.RestoreTerminal(os.Stdin.Fd(), state)
+			}
+			stdin.Close()
+		}()
+		for {
+			e, err := events.Recv()
+			if err != nil {
+				fatal(err.Error(), 1)
+			}
+			if e.Id == id && e.Type == "exit" && e.Pid == pid {
+				os.Exit(int(e.Status))
+			}
+		}
+	},
+}
+
 var startCommand = cli.Command{
 	Name:  "start",
 	Usage: "start a container",
@@ -110,25 +192,26 @@ var startCommand = cli.Command{
 			BundlePath: bpath,
 			Checkpoint: context.String("checkpoint"),
 		}
+		resp, err := c.CreateContainer(netcontext.Background(), r)
+		if err != nil {
+			fatal(err.Error(), 1)
+		}
 		if context.Bool("attach") {
 			mkterm, err := readTermSetting(bpath)
 			if err != nil {
 				fatal(err.Error(), 1)
 			}
 			if mkterm {
-				if err := attachTty(&r.Console); err != nil {
+				s, err := term.SetRawTerminal(os.Stdin.Fd())
-					fatal(err.Error(), 1)
-				}
-			} else {
-				if err := attachStdio(&r.Stdin, &r.Stdout, &r.Stderr); err != nil {
-					fatal(err.Error(), 1)
-				}
-			}
-		}
-		resp, err := c.CreateContainer(netcontext.Background(), r)
 				if err != nil {
 					fatal(err.Error(), 1)
 				}
+				state = s
+			}
+			if err := attachStdio(resp.Stdin, resp.Stdout, resp.Stderr); err != nil {
+				fatal(err.Error(), 1)
+			}
+		}
 		if context.Bool("attach") {
 			restoreAndCloseStdin := func() {
 				if state != nil {
@@ -146,13 +229,11 @@ var startCommand = cli.Command{
 					restoreAndCloseStdin()
 					fatal(err.Error(), 1)
 				}
-				if e.Id == id && e.Type == "exit" {
+				if e.Id == id && e.Type == "exit" && e.Pid == "init" {
 					restoreAndCloseStdin()
 					os.Exit(int(e.Status))
 				}
 			}
-		} else {
-			fmt.Println(resp.Pid)
 		}
 	},
 }
@@ -177,69 +258,24 @@ func readTermSetting(path string) (bool, error) {
 	return spec.Process.Terminal, nil
 }
 
-func attachTty(consolePath *string) error {
+func attachStdio(stdins, stdout, stderr string) error {
-	console, err := libcontainer.NewConsole(os.Getuid(), os.Getgid())
+	stdinf, err := os.OpenFile(stdins, syscall.O_RDWR, 0)
 	if err != nil {
 		return err
 	}
-	*consolePath = console.Path()
+	stdin = stdinf
-	stdin = console
-	go func() {
-		io.Copy(os.Stdout, console)
-		console.Close()
-	}()
-	s, err := term.SetRawTerminal(os.Stdin.Fd())
-	if err != nil {
-		return err
-	}
-	state = s
-	return nil
-}
 
-func attachStdio(stdins, stdout, stderr *string) error {
+	stdoutf, err := os.OpenFile(stdout, syscall.O_RDWR, 0)
-	dir, err := ioutil.TempDir("", "ctr-")
 	if err != nil {
 		return err
 	}
-	for _, p := range []struct {
+	go io.Copy(os.Stdout, stdoutf)
-		path string
+
-		flag int
+	stderrf, err := os.OpenFile(stderr, syscall.O_RDWR, 0)
-		done func(f *os.File)
-	}{
-		{
-			path: filepath.Join(dir, "stdin"),
-			flag: syscall.O_RDWR,
-			done: func(f *os.File) {
-				*stdins = filepath.Join(dir, "stdin")
-				stdin = f
-			},
-		},
-		{
-			path: filepath.Join(dir, "stdout"),
-			flag: syscall.O_RDWR,
-			done: func(f *os.File) {
-				*stdout = filepath.Join(dir, "stdout")
-				go io.Copy(os.Stdout, f)
-			},
-		},
-		{
-			path: filepath.Join(dir, "stderr"),
-			flag: syscall.O_RDWR,
-			done: func(f *os.File) {
-				*stderr = filepath.Join(dir, "stderr")
-				go io.Copy(os.Stderr, f)
-			},
-		},
-	} {
-		if err := syscall.Mkfifo(p.path, 0755); err != nil {
-			return fmt.Errorf("mkfifo: %s %v", p.path, err)
-		}
-		f, err := os.OpenFile(p.path, p.flag, 0)
 	if err != nil {
-			return fmt.Errorf("open: %s %v", p.path, err)
+		return err
-		}
-		p.done(f)
 	}
+	go io.Copy(os.Stderr, stderrf)
 	return nil
 }
 
@@ -247,7 +283,7 @@ var killCommand = cli.Command{
 	Name:  "kill",
 	Usage: "send a signal to a container or its processes",
 	Flags: []cli.Flag{
-		cli.IntFlag{
+		cli.StringFlag{
 			Name:  "pid,p",
 			Usage: "pid of the process to signal within the container",
 		},
@@ -265,7 +301,7 @@ var killCommand = cli.Command{
 		c := getClient(context)
 		if _, err := c.Signal(netcontext.Background(), &types.SignalRequest{
 			Id:     id,
-			Pid:    uint32(context.Int("pid")),
+			Pid:    context.String("pid"),
 			Signal: uint32(context.Int("signal")),
 		}); err != nil {
 			fatal(err.Error(), 1)
@@ -308,6 +344,8 @@ var execCommand = cli.Command{
 		},
 	},
 	Action: func(context *cli.Context) {
+		panic("not implemented")
+		/*
 			p := &types.AddProcessRequest{
 				Args:     context.Args(),
 				Cwd:      context.String("cwd"),
@@ -357,6 +395,7 @@ var execCommand = cli.Command{
 								}
 					}
 				}
+		*/
 	},
 }
 

ctr/events.go

@@ -27,7 +27,7 @@ var eventsCommand = cli.Command{
 			if err != nil {
 				fatal(err.Error(), 1)
 			}
-			fmt.Fprintf(w, "%s\t%s\t%d\t%d\n", e.Type, e.Id, e.Pid, e.Status)
+			fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", e.Type, e.Id, e.Pid, e.Status)
 			w.Flush()
 		}
 	},

linux/linux.go

@@ -1,955 +0,0 @@
-// +build libcontainer
-
-package linux
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	goruntime "runtime"
-	"strconv"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/docker/containerd/runtime"
-	"github.com/opencontainers/runc/libcontainer"
-	"github.com/opencontainers/runc/libcontainer/configs"
-	_ "github.com/opencontainers/runc/libcontainer/nsenter"
-	"github.com/opencontainers/runc/libcontainer/seccomp"
-	"github.com/opencontainers/specs"
-)
-
-const (
-	RLIMIT_CPU        = iota // CPU time in sec
-	RLIMIT_FSIZE             // Maximum filesize
-	RLIMIT_DATA              // max data size
-	RLIMIT_STACK             // max stack size
-	RLIMIT_CORE              // max core file size
-	RLIMIT_RSS               // max resident set size
-	RLIMIT_NPROC             // max number of processes
-	RLIMIT_NOFILE            // max number of open files
-	RLIMIT_MEMLOCK           // max locked-in-memory address space
-	RLIMIT_AS                // address space limit
-	RLIMIT_LOCKS             // maximum file locks held
-	RLIMIT_SIGPENDING        // max number of pending signals
-	RLIMIT_MSGQUEUE          // maximum bytes in POSIX mqueues
-	RLIMIT_NICE              // max nice prio allowed to raise to
-	RLIMIT_RTPRIO            // maximum realtime priority
-	RLIMIT_RTTIME            // timeout for RT tasks in us
-)
-
-var rlimitMap = map[string]int{
-	"RLIMIT_CPU":       RLIMIT_CPU,
-	"RLIMIT_FSIZE":     RLIMIT_FSIZE,
-	"RLIMIT_DATA":      RLIMIT_DATA,
-	"RLIMIT_STACK":     RLIMIT_STACK,
-	"RLIMIT_CORE":      RLIMIT_CORE,
-	"RLIMIT_RSS":       RLIMIT_RSS,
-	"RLIMIT_NPROC":     RLIMIT_NPROC,
-	"RLIMIT_NOFILE":    RLIMIT_NOFILE,
-	"RLIMIT_MEMLOCK":   RLIMIT_MEMLOCK,
-	"RLIMIT_AS":        RLIMIT_AS,
-	"RLIMIT_LOCKS":     RLIMIT_LOCKS,
-	"RLIMIT_SGPENDING": RLIMIT_SIGPENDING,
-	"RLIMIT_MSGQUEUE":  RLIMIT_MSGQUEUE,
-	"RLIMIT_NICE":      RLIMIT_NICE,
-	"RLIMIT_RTPRIO":    RLIMIT_RTPRIO,
-	"RLIMIT_RTTIME":    RLIMIT_RTTIME,
-}
-
-func strToRlimit(key string) (int, error) {
-	rl, ok := rlimitMap[key]
-	if !ok {
-		return 0, fmt.Errorf("Wrong rlimit value: %s", key)
-	}
-	return rl, nil
-}
-
-const wildcard = -1
-
-var allowedDevices = []*configs.Device{
-	// allow mknod for any device
-	{
-		Type:        'c',
-		Major:       wildcard,
-		Minor:       wildcard,
-		Permissions: "m",
-	},
-	{
-		Type:        'b',
-		Major:       wildcard,
-		Minor:       wildcard,
-		Permissions: "m",
-	},
-	{
-		Path:        "/dev/console",
-		Type:        'c',
-		Major:       5,
-		Minor:       1,
-		Permissions: "rwm",
-	},
-	{
-		Path:        "/dev/tty0",
-		Type:        'c',
-		Major:       4,
-		Minor:       0,
-		Permissions: "rwm",
-	},
-	{
-		Path:        "/dev/tty1",
-		Type:        'c',
-		Major:       4,
-		Minor:       1,
-		Permissions: "rwm",
-	},
-	// /dev/pts/ - pts namespaces are "coming soon"
-	{
-		Path:        "",
-		Type:        'c',
-		Major:       136,
-		Minor:       wildcard,
-		Permissions: "rwm",
-	},
-	{
-		Path:        "",
-		Type:        'c',
-		Major:       5,
-		Minor:       2,
-		Permissions: "rwm",
-	},
-	// tuntap
-	{
-		Path:        "",
-		Type:        'c',
-		Major:       10,
-		Minor:       200,
-		Permissions: "rwm",
-	},
-}
-
-var namespaceMapping = map[specs.NamespaceType]configs.NamespaceType{
-	specs.PIDNamespace:     configs.NEWPID,
-	specs.NetworkNamespace: configs.NEWNET,
-	specs.MountNamespace:   configs.NEWNS,
-	specs.UserNamespace:    configs.NEWUSER,
-	specs.IPCNamespace:     configs.NEWIPC,
-	specs.UTSNamespace:     configs.NEWUTS,
-}
-
-var mountPropagationMapping = map[string]int{
-	"rprivate": syscall.MS_PRIVATE | syscall.MS_REC,
-	"private":  syscall.MS_PRIVATE,
-	"rslave":   syscall.MS_SLAVE | syscall.MS_REC,
-	"slave":    syscall.MS_SLAVE,
-	"rshared":  syscall.MS_SHARED | syscall.MS_REC,
-	"shared":   syscall.MS_SHARED,
-	"":         syscall.MS_PRIVATE | syscall.MS_REC,
-}
-
-func init() {
-	if len(os.Args) > 1 && os.Args[1] == "init" {
-		goruntime.GOMAXPROCS(1)
-		goruntime.LockOSThread()
-		factory, _ := libcontainer.New("")
-		if err := factory.StartInitialization(); err != nil {
-			fmt.Fprint(os.Stderr, err)
-			os.Exit(1)
-		}
-		panic("--this line should have never been executed, congratulations--")
-	}
-}
-
-type libcontainerProcess struct {
-	process *libcontainer.Process
-	spec    specs.Process
-}
-
-// change interface to support an error
-func (p *libcontainerProcess) Pid() (int, error) {
-	pid, err := p.process.Pid()
-	if err != nil {
-		return -1, err
-	}
-	return pid, nil
-}
-
-func (p *libcontainerProcess) Spec() specs.Process {
-	return p.spec
-}
-
-func (p *libcontainerProcess) Signal(s os.Signal) error {
-	return p.process.Signal(s)
-}
-
-func (p *libcontainerProcess) Close() error {
-	// in close we always need to call wait to close/flush any pipes
-	p.process.Wait()
-	// explicitly close any open fd on the process
-	for _, cl := range []interface{}{
-		p.process.Stderr,
-		p.process.Stdout,
-		p.process.Stdin,
-	} {
-		if cl != nil {
-			if c, ok := cl.(io.Closer); ok {
-				c.Close()
-			}
-		}
-	}
-	return nil
-}
-
-type libcontainerContainer struct {
-	c                   libcontainer.Container
-	initProcess         *libcontainerProcess
-	additionalProcesses map[int]*libcontainerProcess
-	exitStatus          int
-	exited              bool
-	path                string
-}
-
-func (c *libcontainerContainer) Checkpoints() ([]runtime.Checkpoint, error) {
-	out := []runtime.Checkpoint{}
-	files, err := ioutil.ReadDir(c.getCheckpointPath(""))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return out, nil
-		}
-		return nil, err
-	}
-	for _, fi := range files {
-		out = append(out, runtime.Checkpoint{
-			Name:      fi.Name(),
-			Timestamp: fi.ModTime(),
-		})
-	}
-	return out, nil
-}
-
-func (c *libcontainerContainer) DeleteCheckpoint(name string) error {
-	path := c.getCheckpointPath(name)
-	if err := os.RemoveAll(path); err != nil {
-		if os.IsNotExist(err) {
-			return runtime.ErrCheckpointNotExists
-		}
-		return err
-	}
-	return nil
-}
-
-func (c *libcontainerContainer) getCheckpointPath(name string) string {
-	return filepath.Join(c.path, "checkpoints", name)
-}
-
-func (c *libcontainerContainer) Checkpoint(cp runtime.Checkpoint) error {
-	opts := c.createCheckpointOpts(cp)
-	if err := os.MkdirAll(filepath.Dir(opts.ImagesDirectory), 0755); err != nil {
-		return err
-	}
-	// mkdir is atomic so if it already exists we can fail
-	if err := os.Mkdir(opts.ImagesDirectory, 0755); err != nil {
-		if os.IsExist(err) {
-			return runtime.ErrCheckpointExists
-		}
-		return err
-	}
-	if err := c.c.Checkpoint(opts); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (c *libcontainerContainer) createCheckpointOpts(cp runtime.Checkpoint) *libcontainer.CriuOpts {
-	opts := libcontainer.CriuOpts{}
-	opts.LeaveRunning = !cp.Exit
-	opts.ShellJob = cp.Shell
-	opts.TcpEstablished = cp.Tcp
-	opts.ExternalUnixConnections = cp.UnixSockets
-	opts.ImagesDirectory = c.getCheckpointPath(cp.Name)
-	return &opts
-}
-
-func (c *libcontainerContainer) Restore(name string) error {
-	path := c.getCheckpointPath(name)
-	var opts libcontainer.CriuOpts
-	opts.ImagesDirectory = path
-	return c.c.Restore(c.initProcess.process, &opts)
-}
-
-func (c *libcontainerContainer) Resume() error {
-	return c.c.Resume()
-}
-
-func (c *libcontainerContainer) Pause() error {
-	return c.c.Pause()
-}
-
-func (c *libcontainerContainer) State() runtime.State {
-	// TODO: what to do with error
-	state, err := c.c.Status()
-	if err != nil {
-		return runtime.State("")
-	}
-	switch state {
-	case libcontainer.Paused, libcontainer.Pausing:
-		return runtime.Paused
-	}
-	return runtime.State("")
-}
-
-func (c *libcontainerContainer) ID() string {
-	return c.c.ID()
-}
-
-func (c *libcontainerContainer) Path() string {
-	return c.path
-}
-
-func (c *libcontainerContainer) Pid() (int, error) {
-	return c.initProcess.Pid()
-}
-
-func (c *libcontainerContainer) Start() error {
-	return c.c.Start(c.initProcess.process)
-}
-
-func (c *libcontainerContainer) SetExited(status int) {
-	c.exitStatus = status
-	// meh
-	c.exited = true
-	c.initProcess.Close()
-}
-
-func (c *libcontainerContainer) Stats() (*runtime.Stat, error) {
-	now := time.Now()
-	stats, err := c.c.Stats()
-	if err != nil {
-		return nil, err
-	}
-	return &runtime.Stat{
-		Timestamp: now,
-		Data:      stats,
-	}, nil
-}
-
-func (c *libcontainerContainer) Delete() error {
-	return c.c.Destroy()
-}
-
-func (c *libcontainerContainer) Processes() ([]runtime.Process, error) {
-	procs := []runtime.Process{
-		c.initProcess,
-	}
-	for _, p := range c.additionalProcesses {
-		procs = append(procs, p)
-	}
-	return procs, nil
-}
-
-func (c *libcontainerContainer) RemoveProcess(pid int) error {
-	proc, ok := c.additionalProcesses[pid]
-	if !ok {
-		return runtime.ErrNotChildProcess
-	}
-	err := proc.Close()
-	delete(c.additionalProcesses, pid)
-	return err
-}
-
-func (c *libcontainerContainer) OOM() (<-chan struct{}, error) {
-	return c.c.NotifyOOM()
-}
-
-func NewRuntime(stateDir string) (runtime.Runtime, error) {
-	f, err := libcontainer.New(stateDir, libcontainer.Cgroupfs, func(l *libcontainer.LinuxFactory) error {
-		//l.CriuPath = context.GlobalString("criu")
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	return &libcontainerRuntime{
-		factory: f,
-	}, nil
-}
-
-type libcontainerRuntime struct {
-	factory libcontainer.Factory
-}
-
-func (r *libcontainerRuntime) Type() string {
-	return "libcontainer"
-}
-
-func (r *libcontainerRuntime) Create(id, bundlePath, consolePath string) (runtime.Container, *runtime.IO, error) {
-	spec, rspec, err := r.loadSpec(
-		filepath.Join(bundlePath, "config.json"),
-		filepath.Join(bundlePath, "runtime.json"),
-	)
-	if err != nil {
-		return nil, nil, err
-	}
-	config, err := r.createLibcontainerConfig(id, bundlePath, spec, rspec)
-	if err != nil {
-		return nil, nil, err
-	}
-	container, err := r.factory.Create(id, config)
-	if err != nil {
-		return nil, nil, fmt.Errorf("create container: %v", err)
-	}
-	process, err := r.newProcess(spec.Process)
-	if err != nil {
-		return nil, nil, err
-	}
-	var rio runtime.IO
-	if spec.Process.Terminal {
-		if err := process.ConsoleFromPath(consolePath); err != nil {
-			return nil, nil, err
-		}
-	} else {
-		uid, err := config.HostUID()
-		if err != nil {
-			return nil, nil, err
-		}
-		i, err := process.InitializeIO(uid)
-		if err != nil {
-			return nil, nil, err
-		}
-		rio.Stdin = i.Stdin
-		rio.Stderr = i.Stderr
-		rio.Stdout = i.Stdout
-	}
-	c := &libcontainerContainer{
-		c:                   container,
-		additionalProcesses: make(map[int]*libcontainerProcess),
-		initProcess: &libcontainerProcess{
-			process: process,
-			spec:    spec.Process,
-		},
-		path: bundlePath,
-	}
-	return c, &rio, nil
-}
-
-func (r *libcontainerRuntime) StartProcess(ci runtime.Container, p specs.Process, consolePath string) (runtime.Process, *runtime.IO, error) {
-	c, ok := ci.(*libcontainerContainer)
-	if !ok {
-		return nil, nil, runtime.ErrInvalidContainerType
-	}
-	process, err := r.newProcess(p)
-	if err != nil {
-		return nil, nil, err
-	}
-	var rio runtime.IO
-	if p.Terminal {
-		if err := process.ConsoleFromPath(consolePath); err != nil {
-			return nil, nil, err
-		}
-	} else {
-		uid, err := c.c.Config().HostUID()
-		if err != nil {
-			return nil, nil, err
-		}
-		i, err := process.InitializeIO(uid)
-		if err != nil {
-			return nil, nil, err
-		}
-		rio.Stdin = i.Stdin
-		rio.Stderr = i.Stderr
-		rio.Stdout = i.Stdout
-	}
-	if err := c.c.Start(process); err != nil {
-		return nil, nil, err
-	}
-	lp := &libcontainerProcess{
-		process: process,
-		spec:    p,
-	}
-	pid, err := process.Pid()
-	if err != nil {
-		return nil, nil, err
-	}
-	c.additionalProcesses[pid] = lp
-	return lp, &rio, nil
-}
-
-// newProcess returns a new libcontainer Process with the arguments from the
-// spec and stdio from the current process.
-func (r *libcontainerRuntime) newProcess(p specs.Process) (*libcontainer.Process, error) {
-	return &libcontainer.Process{
-		Args: p.Args,
-		Env:  p.Env,
-		// TODO: fix libcontainer's API to better support uid/gid in a typesafe way.
-		User: fmt.Sprintf("%d:%d", p.User.UID, p.User.GID),
-		Cwd:  p.Cwd,
-	}, nil
-}
-
-// loadSpec loads the specification from the provided path.
-// If the path is empty then the default path will be "config.json"
-func (r *libcontainerRuntime) loadSpec(cPath, rPath string) (spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec, err error) {
-	cf, err := os.Open(cPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil, fmt.Errorf("JSON specification file at %s not found", cPath)
-		}
-		return spec, rspec, err
-	}
-	defer cf.Close()
-
-	rf, err := os.Open(rPath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil, fmt.Errorf("JSON runtime config file at %s not found", rPath)
-		}
-		return spec, rspec, err
-	}
-	defer rf.Close()
-
-	if err = json.NewDecoder(cf).Decode(&spec); err != nil {
-		return spec, rspec, fmt.Errorf("unmarshal %s: %v", cPath, err)
-	}
-	if err = json.NewDecoder(rf).Decode(&rspec); err != nil {
-		return spec, rspec, fmt.Errorf("unmarshal %s: %v", rPath, err)
-	}
-	return spec, rspec, r.checkSpecVersion(spec)
-}
-
-// checkSpecVersion makes sure that the spec version matches runc's while we are in the initial
-// development period.  It is better to hard fail than have missing fields or options in the spec.
-func (r *libcontainerRuntime) checkSpecVersion(s *specs.LinuxSpec) error {
-	if s.Version != specs.Version {
-		return fmt.Errorf("spec version is not compatible with implemented version %q: spec %q", specs.Version, s.Version)
-	}
-	return nil
-}
-
-func (r *libcontainerRuntime) createLibcontainerConfig(cgroupName, bundlePath string, spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec) (*configs.Config, error) {
-	rootfsPath := spec.Root.Path
-	if !filepath.IsAbs(rootfsPath) {
-		rootfsPath = filepath.Join(bundlePath, rootfsPath)
-	}
-	config := &configs.Config{
-		Rootfs:       rootfsPath,
-		Capabilities: spec.Linux.Capabilities,
-		Readonlyfs:   spec.Root.Readonly,
-		Hostname:     spec.Hostname,
-	}
-	for _, ns := range rspec.Linux.Namespaces {
-		t, exists := namespaceMapping[ns.Type]
-		if !exists {
-			return nil, fmt.Errorf("namespace %q does not exist", ns)
-		}
-		config.Namespaces.Add(t, ns.Path)
-	}
-	if config.Namespaces.Contains(configs.NEWNET) {
-		config.Networks = []*configs.Network{
-			{
-				Type: "loopback",
-			},
-		}
-	}
-	for _, mp := range spec.Mounts {
-		m, ok := rspec.Mounts[mp.Name]
-		if !ok {
-			return nil, fmt.Errorf("Mount with Name %q not found in runtime config", mp.Name)
-		}
-		config.Mounts = append(config.Mounts, r.createLibcontainerMount(bundlePath, mp.Path, m))
-	}
-
-	// Convert rootfs propagation flag
-	if rspec.Linux.RootfsPropagation != "" {
-		_, pflags, _ := parseMountOptions([]string{rspec.Linux.RootfsPropagation})
-		if len(pflags) == 1 {
-			config.RootPropagation = pflags[0]
-		}
-	}
-
-	if err := r.createDevices(rspec, config); err != nil {
-		return nil, err
-	}
-	if err := r.setupUserNamespace(rspec, config); err != nil {
-		return nil, err
-	}
-	for _, rlimit := range rspec.Linux.Rlimits {
-		rl, err := r.createLibContainerRlimit(rlimit)
-		if err != nil {
-			return nil, err
-		}
-		config.Rlimits = append(config.Rlimits, rl)
-	}
-	c, err := r.createCgroupConfig(cgroupName, rspec, config.Devices)
-	if err != nil {
-		return nil, err
-	}
-	config.Cgroups = c
-	if config.Readonlyfs {
-		r.setReadonly(config)
-		config.MaskPaths = []string{
-			"/proc/kcore",
-		}
-		config.ReadonlyPaths = []string{
-			"/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus",
-		}
-	}
-	seccomp, err := r.setupSeccomp(&rspec.Linux.Seccomp)
-	if err != nil {
-		return nil, err
-	}
-	config.Seccomp = seccomp
-	config.Sysctl = rspec.Linux.Sysctl
-	config.ProcessLabel = rspec.Linux.SelinuxProcessLabel
-	config.AppArmorProfile = rspec.Linux.ApparmorProfile
-	for _, g := range spec.Process.User.AdditionalGids {
-		config.AdditionalGroups = append(config.AdditionalGroups, strconv.FormatUint(uint64(g), 10))
-	}
-	r.createHooks(rspec, config)
-	config.Version = specs.Version
-	return config, nil
-}
-
-func (r *libcontainerRuntime) createLibcontainerMount(cwd, dest string, m specs.Mount) *configs.Mount {
-	flags, pgflags, data := parseMountOptions(m.Options)
-	source := m.Source
-	if m.Type == "bind" {
-		if !filepath.IsAbs(source) {
-			source = filepath.Join(cwd, m.Source)
-		}
-	}
-	return &configs.Mount{
-		Device:           m.Type,
-		Source:           source,
-		Destination:      dest,
-		Data:             data,
-		Flags:            flags,
-		PropagationFlags: pgflags,
-	}
-}
-
-func (rt *libcontainerRuntime) createCgroupConfig(name string, spec *specs.LinuxRuntimeSpec, devices []*configs.Device) (*configs.Cgroup, error) {
-	cr := &configs.Cgroup{
-		Name:   name,
-		Parent: "/containerd",
-	}
-	c := &configs.Resources{
-		AllowedDevices: append(devices, allowedDevices...),
-	}
-	cr.Resources = c
-	r := spec.Linux.Resources
-	if r.Memory != nil {
-		if r.Memory.Limit != nil {
-			c.Memory = int64(*r.Memory.Limit)
-		}
-		if r.Memory.Reservation != nil {
-			c.MemoryReservation = int64(*r.Memory.Reservation)
-		}
-		if r.Memory.Swap != nil {
-			c.MemorySwap = int64(*r.Memory.Swap)
-		}
-		if r.Memory.Kernel != nil {
-			c.KernelMemory = int64(*r.Memory.Kernel)
-		}
-		if r.Memory.Swappiness != nil {
-			c.MemorySwappiness = int64(*r.Memory.Swappiness)
-		}
-	}
-	if r.CPU != nil {
-		if r.CPU.Shares != nil {
-			c.CpuShares = int64(*r.CPU.Shares)
-		}
-		if r.CPU.Quota != nil {
-			c.CpuQuota = int64(*r.CPU.Quota)
-		}
-		if r.CPU.Period != nil {
-			c.CpuPeriod = int64(*r.CPU.Period)
-		}
-		if r.CPU.RealtimeRuntime != nil {
-			c.CpuRtRuntime = int64(*r.CPU.RealtimeRuntime)
-		}
-		if r.CPU.RealtimePeriod != nil {
-			c.CpuRtPeriod = int64(*r.CPU.RealtimePeriod)
-		}
-		if r.CPU.Cpus != nil {
-			c.CpusetCpus = *r.CPU.Cpus
-		}
-		if r.CPU.Mems != nil {
-			c.CpusetMems = *r.CPU.Mems
-		}
-	}
-	if r.BlockIO != nil {
-		if r.BlockIO.Weight != nil {
-			c.BlkioWeight = *r.BlockIO.Weight
-		}
-		if r.BlockIO.LeafWeight != nil {
-			c.BlkioLeafWeight = *r.BlockIO.LeafWeight
-		}
-	}
-	for _, wd := range r.BlockIO.WeightDevice {
-		weightDevice := configs.NewWeightDevice(wd.Major, wd.Minor, *wd.Weight, *wd.LeafWeight)
-		c.BlkioWeightDevice = append(c.BlkioWeightDevice, weightDevice)
-	}
-	for _, td := range r.BlockIO.ThrottleReadBpsDevice {
-		throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate)
-		c.BlkioThrottleReadBpsDevice = append(c.BlkioThrottleReadBpsDevice, throttleDevice)
-	}
-	for _, td := range r.BlockIO.ThrottleWriteBpsDevice {
-		throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate)
-		c.BlkioThrottleWriteBpsDevice = append(c.BlkioThrottleWriteBpsDevice, throttleDevice)
-	}
-	for _, td := range r.BlockIO.ThrottleReadIOPSDevice {
-		throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate)
-		c.BlkioThrottleReadIOPSDevice = append(c.BlkioThrottleReadIOPSDevice, throttleDevice)
-	}
-	for _, td := range r.BlockIO.ThrottleWriteIOPSDevice {
-		throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate)
-		c.BlkioThrottleWriteIOPSDevice = append(c.BlkioThrottleWriteIOPSDevice, throttleDevice)
-	}
-	for _, l := range r.HugepageLimits {
-		c.HugetlbLimit = append(c.HugetlbLimit, &configs.HugepageLimit{
-			Pagesize: *l.Pagesize,
-			Limit:    *l.Limit,
-		})
-	}
-	c.OomKillDisable = r.DisableOOMKiller != nil && *r.DisableOOMKiller
-	if r.Network != nil {
-		c.NetClsClassid = r.Network.ClassID
-		for _, m := range r.Network.Priorities {
-			c.NetPrioIfpriomap = append(c.NetPrioIfpriomap, &configs.IfPrioMap{
-				Interface: m.Name,
-				Priority:  int64(m.Priority),
-			})
-		}
-	}
-	return cr, nil
-}
-
-func (r *libcontainerRuntime) createDevices(spec *specs.LinuxRuntimeSpec, config *configs.Config) error {
-	for _, d := range spec.Linux.Devices {
-		device := &configs.Device{
-			Type:        d.Type,
-			Path:        d.Path,
-			Major:       d.Major,
-			Minor:       d.Minor,
-			Permissions: d.Permissions,
-			FileMode:    d.FileMode,
-			Uid:         d.UID,
-			Gid:         d.GID,
-		}
-		config.Devices = append(config.Devices, device)
-	}
-	return nil
-}
-
-func (r *libcontainerRuntime) setReadonly(config *configs.Config) {
-	for _, m := range config.Mounts {
-		if m.Device == "sysfs" {
-			m.Flags |= syscall.MS_RDONLY
-		}
-	}
-}
-
-func (r *libcontainerRuntime) setupUserNamespace(spec *specs.LinuxRuntimeSpec, config *configs.Config) error {
-	if len(spec.Linux.UIDMappings) == 0 {
-		return nil
-	}
-	config.Namespaces.Add(configs.NEWUSER, "")
-	create := func(m specs.IDMapping) configs.IDMap {
-		return configs.IDMap{
-			HostID:      int(m.HostID),
-			ContainerID: int(m.ContainerID),
-			Size:        int(m.Size),
-		}
-	}
-	for _, m := range spec.Linux.UIDMappings {
-		config.UidMappings = append(config.UidMappings, create(m))
-	}
-	for _, m := range spec.Linux.GIDMappings {
-		config.GidMappings = append(config.GidMappings, create(m))
-	}
-	rootUID, err := config.HostUID()
-	if err != nil {
-		return err
-	}
-	rootGID, err := config.HostGID()
-	if err != nil {
-		return err
-	}
-	for _, node := range config.Devices {
-		node.Uid = uint32(rootUID)
-		node.Gid = uint32(rootGID)
-	}
-	return nil
-}
-
-func (r *libcontainerRuntime) createLibContainerRlimit(rlimit specs.Rlimit) (configs.Rlimit, error) {
-	rl, err := strToRlimit(rlimit.Type)
-	if err != nil {
-		return configs.Rlimit{}, err
-	}
-	return configs.Rlimit{
-		Type: rl,
-		Hard: uint64(rlimit.Hard),
-		Soft: uint64(rlimit.Soft),
-	}, nil
-}
-
-// parseMountOptions parses the string and returns the flags, propagation
-// flags and any mount data that it contains.
-func parseMountOptions(options []string) (int, []int, string) {
-	var (
-		flag   int
-		pgflag []int
-		data   []string
-	)
-	flags := map[string]struct {
-		clear bool
-		flag  int
-	}{
-		"async":         {true, syscall.MS_SYNCHRONOUS},
-		"atime":         {true, syscall.MS_NOATIME},
-		"bind":          {false, syscall.MS_BIND},
-		"defaults":      {false, 0},
-		"dev":           {true, syscall.MS_NODEV},
-		"diratime":      {true, syscall.MS_NODIRATIME},
-		"dirsync":       {false, syscall.MS_DIRSYNC},
-		"exec":          {true, syscall.MS_NOEXEC},
-		"mand":          {false, syscall.MS_MANDLOCK},
-		"noatime":       {false, syscall.MS_NOATIME},
-		"nodev":         {false, syscall.MS_NODEV},
-		"nodiratime":    {false, syscall.MS_NODIRATIME},
-		"noexec":        {false, syscall.MS_NOEXEC},
-		"nomand":        {true, syscall.MS_MANDLOCK},
-		"norelatime":    {true, syscall.MS_RELATIME},
-		"nostrictatime": {true, syscall.MS_STRICTATIME},
-		"nosuid":        {false, syscall.MS_NOSUID},
-		"rbind":         {false, syscall.MS_BIND | syscall.MS_REC},
-		"relatime":      {false, syscall.MS_RELATIME},
-		"remount":       {false, syscall.MS_REMOUNT},
-		"ro":            {false, syscall.MS_RDONLY},
-		"rw":            {true, syscall.MS_RDONLY},
-		"strictatime":   {false, syscall.MS_STRICTATIME},
-		"suid":          {true, syscall.MS_NOSUID},
-		"sync":          {false, syscall.MS_SYNCHRONOUS},
-	}
-	propagationFlags := map[string]struct {
-		clear bool
-		flag  int
-	}{
-		"private":     {false, syscall.MS_PRIVATE},
-		"shared":      {false, syscall.MS_SHARED},
-		"slave":       {false, syscall.MS_SLAVE},
-		"unbindable":  {false, syscall.MS_UNBINDABLE},
-		"rprivate":    {false, syscall.MS_PRIVATE | syscall.MS_REC},
-		"rshared":     {false, syscall.MS_SHARED | syscall.MS_REC},
-		"rslave":      {false, syscall.MS_SLAVE | syscall.MS_REC},
-		"runbindable": {false, syscall.MS_UNBINDABLE | syscall.MS_REC},
-	}
-	for _, o := range options {
-		// If the option does not exist in the flags table or the flag
-		// is not supported on the platform,
-		// then it is a data value for a specific fs type
-		if f, exists := flags[o]; exists && f.flag != 0 {
-			if f.clear {
-				flag &= ^f.flag
-			} else {
-				flag |= f.flag
-			}
-		} else if f, exists := propagationFlags[o]; exists && f.flag != 0 {
-			pgflag = append(pgflag, f.flag)
-		} else {
-			data = append(data, o)
-		}
-	}
-	return flag, pgflag, strings.Join(data, ",")
-}
-
-func (r *libcontainerRuntime) setupSeccomp(config *specs.Seccomp) (*configs.Seccomp, error) {
-	if config == nil {
-		return nil, nil
-	}
-
-	// No default action specified, no syscalls listed, assume seccomp disabled
-	if config.DefaultAction == "" && len(config.Syscalls) == 0 {
-		return nil, nil
-	}
-
-	newConfig := new(configs.Seccomp)
-	newConfig.Syscalls = []*configs.Syscall{}
-
-	if len(config.Architectures) > 0 {
-		newConfig.Architectures = []string{}
-		for _, arch := range config.Architectures {
-			newArch, err := seccomp.ConvertStringToArch(string(arch))
-			if err != nil {
-				return nil, err
-			}
-			newConfig.Architectures = append(newConfig.Architectures, newArch)
-		}
-	}
-
-	// Convert default action from string representation
-	newDefaultAction, err := seccomp.ConvertStringToAction(string(config.DefaultAction))
-	if err != nil {
-		return nil, err
-	}
-	newConfig.DefaultAction = newDefaultAction
-
-	// Loop through all syscall blocks and convert them to libcontainer format
-	for _, call := range config.Syscalls {
-		newAction, err := seccomp.ConvertStringToAction(string(call.Action))
-		if err != nil {
-			return nil, err
-		}
-
-		newCall := configs.Syscall{
-			Name:   call.Name,
-			Action: newAction,
-			Args:   []*configs.Arg{},
-		}
-
-		// Loop through all the arguments of the syscall and convert them
-		for _, arg := range call.Args {
-			newOp, err := seccomp.ConvertStringToOperator(string(arg.Op))
-			if err != nil {
-				return nil, err
-			}
-
-			newArg := configs.Arg{
-				Index:    arg.Index,
-				Value:    arg.Value,
-				ValueTwo: arg.ValueTwo,
-				Op:       newOp,
-			}
-
-			newCall.Args = append(newCall.Args, &newArg)
-		}
-
-		newConfig.Syscalls = append(newConfig.Syscalls, &newCall)
-	}
-
-	return newConfig, nil
-}
-
-func (r *libcontainerRuntime) createHooks(rspec *specs.LinuxRuntimeSpec, config *configs.Config) {
-	config.Hooks = &configs.Hooks{}
-	for _, h := range rspec.Hooks.Prestart {
-		cmd := configs.Command{
-			Path: h.Path,
-			Args: h.Args,
-			Env:  h.Env,
-		}
-		config.Hooks.Prestart = append(config.Hooks.Prestart, configs.NewCommandHook(cmd))
-	}
-	for _, h := range rspec.Hooks.Poststop {
-		cmd := configs.Command{
-			Path: h.Path,
-			Args: h.Args,
-			Env:  h.Env,
-		}
-		config.Hooks.Poststop = append(config.Hooks.Poststop, configs.NewCommandHook(cmd))
-	}
-}

runc/runc.go

@@ -1,235 +0,0 @@
-// +build runc
-
-package runc
-
-import (
-	"encoding/json"
-	"errors"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-
-	"github.com/docker/containerd/runtime"
-	"github.com/opencontainers/specs"
-)
-
-func NewRuntime(stateDir string) (runtime.Runtime, error) {
-	return &runcRuntime{
-		stateDir: stateDir,
-	}, nil
-}
-
-type runcContainer struct {
-	id          string
-	path        string
-	stateDir    string
-	exitStatus  int
-	processes   map[int]*runcProcess
-	initProcess *runcProcess
-}
-
-func (c *runcContainer) ID() string {
-	return c.id
-}
-
-func (c *runcContainer) Start() error {
-	return c.initProcess.cmd.Start()
-}
-
-func (c *runcContainer) Stats() (*runtime.Stat, error) {
-	return nil, errors.New("containerd: runc does not support stats in containerd")
-}
-
-func (c *runcContainer) Path() string {
-	return c.path
-}
-
-func (c *runcContainer) Pid() (int, error) {
-	return c.initProcess.cmd.Process.Pid, nil
-}
-
-func (c *runcContainer) SetExited(status int) {
-	c.exitStatus = status
-}
-
-// noop for runc
-func (c *runcContainer) Delete() error {
-	return nil
-}
-
-func (c *runcContainer) Processes() ([]runtime.Process, error) {
-	procs := []runtime.Process{
-		c.initProcess,
-	}
-	for _, p := range c.processes {
-		procs = append(procs, p)
-	}
-	return procs, nil
-}
-
-func (c *runcContainer) RemoveProcess(pid int) error {
-	if _, ok := c.processes[pid]; !ok {
-		return runtime.ErrNotChildProcess
-	}
-	delete(c.processes, pid)
-	return nil
-}
-
-func (c *runcContainer) State() runtime.State {
-	// TODO: how to do this with runc
-	return runtime.State{
-		Status: runtime.Running,
-	}
-}
-
-func (c *runcContainer) Resume() error {
-	return c.newCommand("resume").Run()
-}
-
-func (c *runcContainer) Pause() error {
-	return c.newCommand("pause").Run()
-}
-
-// TODO: pass arguments
-func (c *runcContainer) Checkpoint(runtime.Checkpoint) error {
-	return c.newCommand("checkpoint").Run()
-}
-
-// TODO: pass arguments
-func (c *runcContainer) Restore(cp string) error {
-	return c.newCommand("restore").Run()
-}
-
-// TODO: pass arguments
-func (c *runcContainer) DeleteCheckpoint(cp string) error {
-	return errors.New("not implemented")
-}
-
-// TODO: implement in runc
-func (c *runcContainer) Checkpoints() ([]runtime.Checkpoint, error) {
-	return nil, errors.New("not implemented")
-}
-
-func (c *runcContainer) OOM() (<-chan struct{}, error) {
-	return nil, errors.New("not implemented")
-}
-
-func (c *runcContainer) newCommand(args ...string) *exec.Cmd {
-	cmd := exec.Command("runc", append([]string{"--root", c.stateDir, "--id", c.id}, args...)...)
-	cmd.Dir = c.path
-	return cmd
-}
-
-type runcProcess struct {
-	cmd  *exec.Cmd
-	spec specs.Process
-}
-
-// pid of the container, not of runc
-func (p *runcProcess) Pid() (int, error) {
-	return p.cmd.Process.Pid, nil
-}
-
-func (p *runcProcess) Spec() specs.Process {
-	return p.spec
-}
-
-func (p *runcProcess) Signal(s os.Signal) error {
-	return p.cmd.Process.Signal(s)
-}
-
-func (p *runcProcess) Close() error {
-	return nil
-}
-
-type runcRuntime struct {
-	stateDir string
-}
-
-func (r *runcRuntime) Type() string {
-	return "runc"
-}
-
-func (r *runcRuntime) Create(id, bundlePath, consolePath string) (runtime.Container, *runtime.IO, error) {
-	var s specs.Spec
-	f, err := os.Open(filepath.Join(bundlePath, "config.json"))
-	if err != nil {
-		return nil, nil, err
-	}
-	defer f.Close()
-
-	if err := json.NewDecoder(f).Decode(&s); err != nil {
-		return nil, nil, err
-	}
-	cmd := exec.Command("runc", "--root", r.stateDir, "--id", id, "start")
-	cmd.Dir = bundlePath
-	i, err := r.createIO(cmd)
-	if err != nil {
-		return nil, nil, err
-	}
-	return &runcContainer{
-		id:       id,
-		path:     bundlePath,
-		stateDir: r.stateDir,
-		initProcess: &runcProcess{
-			cmd:  cmd,
-			spec: s.Process,
-		},
-		processes: make(map[int]*runcProcess),
-	}, i, nil
-}
-
-func (r *runcRuntime) createIO(cmd *exec.Cmd) (*runtime.IO, error) {
-	w, err := cmd.StdinPipe()
-	if err != nil {
-		return nil, err
-	}
-	ro, err := cmd.StdoutPipe()
-	if err != nil {
-		return nil, err
-	}
-	re, err := cmd.StderrPipe()
-	if err != nil {
-		return nil, err
-	}
-	return &runtime.IO{
-		Stdin:  w,
-		Stdout: ro,
-		Stderr: re,
-	}, nil
-}
-
-func (r *runcRuntime) StartProcess(ci runtime.Container, p specs.Process, consolePath string) (runtime.Process, *runtime.IO, error) {
-	c, ok := ci.(*runcContainer)
-	if !ok {
-		return nil, nil, runtime.ErrInvalidContainerType
-	}
-	f, err := ioutil.TempFile("", "containerd")
-	if err != nil {
-		return nil, nil, err
-	}
-	err = json.NewEncoder(f).Encode(p)
-	f.Close()
-	if err != nil {
-		return nil, nil, err
-	}
-	cmd := c.newCommand("exec", f.Name())
-	i, err := r.createIO(cmd)
-	if err != nil {
-		return nil, nil, err
-	}
-	process := &runcProcess{
-		cmd:  cmd,
-		spec: p,
-	}
-	if err := cmd.Start(); err != nil {
-		return nil, nil, err
-	}
-	pid, err := process.Pid()
-	if err != nil {
-		return nil, nil, err
-	}
-	c.processes[pid] = process
-	return process, i, nil
-}

runtime/container.go

@@ -10,9 +10,27 @@ import (
 
 type Process interface {
 	io.Closer
-	Pid() (int, error)
+
+	// ID of the process.
+	// This is either "init" when it is the container's init process or
+	// it is a user provided id for the process similar to the container id
+	ID() string
+	// Stdin returns the path the the processes stdin fifo
+	Stdin() string
+	// Stdout returns the path the the processes stdout fifo
+	Stdout() string
+	// Stderr returns the path the the processes stderr fifo
+	Stderr() string
+	// ExitFD returns the fd the provides an event when the process exits
+	ExitFD() int
+	// ExitStatus returns the exit status of the process or an error if it
+	// has not exited
+	ExitStatus() (int, error)
 	Spec() specs.Process
+	// Signal sends the provided signal to the process
 	Signal(os.Signal) error
+	// Container returns the container that the process belongs to
+	Container() Container
 }
 
 type State string
@@ -77,20 +95,16 @@ type Checkpoint struct {
 type Container interface {
 	// ID returns the container ID
 	ID() string
-	// Start starts the init process of the container
-	Start() error
 	// Path returns the path to the bundle
 	Path() string
-	// Pid returns the container's init process id
+	// Start starts the init process of the container
-	Pid() (int, error)
+	Start() (Process, error)
-	// SetExited sets the exit status of the container after its init dies
+	// Delete removes the container's state and any resources
-	SetExited(status int)
-	// Delete deletes the container
 	Delete() error
+	// Pid returns the container's init process id
+	// Pid() (int, error)
 	// Processes returns all the containers processes that have been added
 	Processes() ([]Process, error)
-	// RemoveProcess removes a specific process for the container because it exited
-	RemoveProcess(pid int) error
 	// State returns the containers runtime state
 	State() State
 	// Resume resumes a paused container
@@ -98,15 +112,15 @@ type Container interface {
 	// Pause pauses a running container
 	Pause() error
 	// Checkpoints returns all the checkpoints for a container
-	Checkpoints() ([]Checkpoint, error)
+	// Checkpoints() ([]Checkpoint, error)
 	// Checkpoint creates a new checkpoint
-	Checkpoint(Checkpoint) error
+	// Checkpoint(Checkpoint) error
 	// DeleteCheckpoint deletes the checkpoint for the provided name
-	DeleteCheckpoint(name string) error
+	// DeleteCheckpoint(name string) error
 	// Restore restores the container to that of the checkpoint provided by name
-	Restore(name string) error
+	// Restore(name string) error
 	// Stats returns realtime container stats and resource information
-	Stats() (*Stat, error)
+	// Stats() (*Stat, error)
 	// OOM signals the channel if the container received an OOM notification
-	OOM() (<-chan struct{}, error)
+	// OOM() (<-chan struct{}, error)
 }

runtime/lib.go

@@ -0,0 +1,172 @@
+package runtime
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/opencontainers/specs"
+)
+
+const (
+	ExitFile       = "exit"
+	ExitStatusFile = "exitStatus"
+	StateFile      = "state.json"
+	InitProcessID  = "init"
+)
+
+type state struct {
+	Bundle string `json:"bundle"`
+}
+
+// New returns a new container
+func New(root, id, bundle string) (Container, error) {
+	c := &container{
+		root:      root,
+		id:        id,
+		bundle:    bundle,
+		processes: make(map[string]*process),
+	}
+	if err := os.Mkdir(filepath.Join(root, id), 0755); err != nil {
+		return nil, err
+	}
+	f, err := os.Create(filepath.Join(root, id, StateFile))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	if err := json.NewEncoder(f).Encode(state{
+		Bundle: bundle,
+	}); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+func Load(root, id string) (Container, error) {
+	var s state
+	f, err := os.Open(filepath.Join(root, id, StateFile))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	if err := json.NewDecoder(f).Decode(&s); err != nil {
+		return nil, err
+	}
+	c := &container{
+		root:      root,
+		id:        id,
+		bundle:    s.Bundle,
+		processes: make(map[string]*process),
+	}
+	dirs, err := ioutil.ReadDir(filepath.Join(root, id))
+	if err != nil {
+		return nil, err
+	}
+	for _, d := range dirs {
+		if !d.IsDir() {
+			continue
+		}
+		pid := d.Name()
+		// TODO: get the process spec from a state file in the process dir
+		p, err := loadProcess(filepath.Join(root, id, pid), pid, c, specs.Process{})
+		if err != nil {
+			if err == ErrProcessExited {
+				logrus.WithField("id", id).WithField("pid", pid).Debug("containerd: process exited while away")
+				// TODO: fire events to do the removal
+				if err := os.RemoveAll(filepath.Join(root, id, pid)); err != nil {
+					logrus.WithField("error", err).Warn("containerd: remove process state")
+				}
+				continue
+			}
+			return nil, err
+		}
+		c.processes[pid] = p
+	}
+	if len(c.processes) == 0 {
+		return nil, ErrContainerExited
+	}
+	return c, nil
+}
+
+type container struct {
+	// path to store runtime state information
+	root      string
+	id        string
+	bundle    string
+	processes map[string]*process
+}
+
+func (c *container) ID() string {
+	return c.id
+}
+
+func (c *container) Path() string {
+	return c.bundle
+}
+
+func (c *container) Start() (Process, error) {
+	processRoot := filepath.Join(c.root, c.id, InitProcessID)
+	if err := os.MkdirAll(processRoot, 0755); err != nil {
+		return nil, err
+	}
+	cmd := exec.Command("containerd-shim", processRoot, c.id)
+	cmd.Dir = c.bundle
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	spec, err := c.readSpec()
+	if err != nil {
+		return nil, err
+	}
+	p, err := newProcess(processRoot, InitProcessID, c, spec.Process)
+	if err != nil {
+		return nil, err
+	}
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+	c.processes[InitProcessID] = p
+	return p, nil
+}
+
+func (c *container) readSpec() (*specs.LinuxSpec, error) {
+	var spec specs.LinuxSpec
+	f, err := os.Open(filepath.Join(c.bundle, "config.json"))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	if err := json.NewDecoder(f).Decode(&spec); err != nil {
+		return nil, err
+	}
+	return &spec, nil
+}
+
+func (c *container) Pause() error {
+	return errNotImplemented
+}
+
+func (c *container) Resume() error {
+	return errNotImplemented
+}
+
+func (c *container) State() State {
+	return Running
+}
+
+func (c *container) Delete() error {
+	return os.RemoveAll(filepath.Join(c.root, c.id))
+}
+
+func (c *container) Processes() ([]Process, error) {
+	out := []Process{}
+	for _, p := range c.processes {
+		out = append(out, p)
+	}
+	return out, nil
+}

runtime/process.go

@@ -0,0 +1,141 @@
+package runtime
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"github.com/opencontainers/specs"
+)
+
+func newProcess(root, id string, c *container, s specs.Process) (*process, error) {
+	p := &process{
+		root:      root,
+		id:        id,
+		container: c,
+		spec:      s,
+	}
+	// create fifo's for the process
+	for name, fd := range map[string]*string{
+		"stdin":  &p.stdin,
+		"stdout": &p.stdout,
+		"stderr": &p.stderr,
+	} {
+		path := filepath.Join(root, name)
+		if err := syscall.Mkfifo(path, 0755); err != nil && !os.IsExist(err) {
+			return nil, err
+		}
+		*fd = path
+	}
+	exit, err := getExitPipe(filepath.Join(root, ExitFile))
+	if err != nil {
+		return nil, err
+	}
+	p.exitPipe = exit
+	return p, nil
+}
+
+func loadProcess(root, id string, c *container, s specs.Process) (*process, error) {
+	p := &process{
+		root:      root,
+		id:        id,
+		container: c,
+		spec:      s,
+		stdin:     filepath.Join(root, "stdin"),
+		stdout:    filepath.Join(root, "stdout"),
+		stderr:    filepath.Join(root, "stderr"),
+	}
+	if _, err := p.ExitStatus(); err != nil {
+		if err == ErrProcessNotExited {
+			exit, err := getExitPipe(filepath.Join(root, ExitFile))
+			if err != nil {
+				return nil, err
+			}
+			p.exitPipe = exit
+			return p, nil
+		}
+		return nil, err
+	}
+	return nil, ErrProcessExited
+}
+
+func getExitPipe(path string) (*os.File, error) {
+	if err := syscall.Mkfifo(path, 0755); err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+	// add NONBLOCK in case the other side has already closed or else
+	// this function would never return
+	return os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
+}
+
+type process struct {
+	root string
+	id   string
+	// stdio fifos
+	stdin  string
+	stdout string
+	stderr string
+
+	exitPipe  *os.File
+	container *container
+	spec      specs.Process
+}
+
+func (p *process) ID() string {
+	return p.id
+}
+
+func (p *process) Container() Container {
+	return p.container
+}
+
+// ExitFD returns the fd of the exit pipe
+func (p *process) ExitFD() int {
+	return int(p.exitPipe.Fd())
+}
+
+func (p *process) ExitStatus() (int, error) {
+	data, err := ioutil.ReadFile(filepath.Join(p.root, ExitStatusFile))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return -1, ErrProcessNotExited
+		}
+		return -1, err
+	}
+	if len(data) == 0 {
+		return -1, ErrProcessNotExited
+	}
+	i, err := strconv.Atoi(string(data))
+	if err != nil {
+		return -1, err
+	}
+	return i, nil
+}
+
+// Signal sends the provided signal to the process
+func (p *process) Signal(s os.Signal) error {
+	return errNotImplemented
+}
+
+func (p *process) Spec() specs.Process {
+	return p.spec
+}
+
+func (p *process) Stdin() string {
+	return p.stdin
+}
+
+func (p *process) Stdout() string {
+	return p.stdout
+}
+
+func (p *process) Stderr() string {
+	return p.stderr
+}
+
+// Close closes any open files and/or resouces on the process
+func (p *process) Close() error {
+	return p.exitPipe.Close()
+}

runtime/runtime.go

@@ -1,10 +1,6 @@
 package runtime
 
-import (
+import "errors"
-	"errors"
-
-	"github.com/opencontainers/specs"
-)
 
 var (
 	ErrNotChildProcess       = errors.New("containerd: not a child process for container")
@@ -13,14 +9,8 @@ var (
 	ErrCheckpointExists      = errors.New("containerd: checkpoint already exists")
 	ErrContainerExited       = errors.New("containerd: container has exited")
 	ErrTerminalsNotSupported = errors.New("containerd: terminals are not supported for runtime")
-)
+	ErrProcessNotExited      = errors.New("containerd: process has not exited")
+	ErrProcessExited         = errors.New("containerd: process has exited")
 
-// Runtime handles containers, containers handle their own actions
+	errNotImplemented = errors.New("containerd: not implemented")
-type Runtime interface {
+)
-	// Type of the runtime
-	Type() string
-	// Create creates a new container initialized but without it starting it
-	Create(id, bundlePath, consolePath string) (Container, *IO, error)
-	// StartProcess adds a new process to the container
-	StartProcess(c Container, p specs.Process, consolePath string) (Process, *IO, error)
-}

supervisor/add_process.go

@@ -1,11 +1,5 @@
 package supervisor
 
-import (
-	"time"
-
-	"github.com/Sirupsen/logrus"
-)
-
 type AddProcessEvent struct {
 	s *Supervisor
 }
@@ -13,6 +7,7 @@ type AddProcessEvent struct {
 // TODO: add this to worker for concurrent starts???  maybe not because of races where the container
 // could be stopped and removed...
 func (h *AddProcessEvent) Handle(e *Event) error {
+	/*
 		start := time.Now()
 		ci, ok := h.s.containers[e.ID]
 		if !ok {
@@ -38,5 +33,6 @@ func (h *AddProcessEvent) Handle(e *Event) error {
 			}
 			h.s.processes[e.Pid].copier = l
 			ExecProcessTimer.UpdateSince(start)
+	*/
 	return nil
 }

supervisor/checkpoint.go

@@ -5,11 +5,14 @@ type CreateCheckpointEvent struct {
 }
 
 func (h *CreateCheckpointEvent) Handle(e *Event) error {
+	/*
 		i, ok := h.s.containers[e.ID]
 		if !ok {
 			return ErrContainerNotFound
 		}
-	return i.container.Checkpoint(*e.Checkpoint)
+	*/
+	return nil
+	// return i.container.Checkpoint(*e.Checkpoint)
 }
 
 type DeleteCheckpointEvent struct {
@@ -17,9 +20,12 @@ type DeleteCheckpointEvent struct {
 }
 
 func (h *DeleteCheckpointEvent) Handle(e *Event) error {
+	/*
 		i, ok := h.s.containers[e.ID]
 		if !ok {
 			return ErrContainerNotFound
 		}
-	return i.container.DeleteCheckpoint(e.Checkpoint.Name)
+	*/
+	return nil
+	// return i.container.DeleteCheckpoint(e.Checkpoint.Name)
 }

supervisor/create.go

@@ -1,6 +1,10 @@
 package supervisor
 
-import "time"
+import (
+	"time"
+
+	"github.com/docker/containerd/runtime"
+)
 
 type StartEvent struct {
 	s *Supervisor
@@ -8,22 +12,17 @@ type StartEvent struct {
 
 func (h *StartEvent) Handle(e *Event) error {
 	start := time.Now()
-	container, io, err := h.s.runtime.Create(e.ID, e.BundlePath, e.Console)
+	container, err := runtime.New(h.s.stateDir, e.ID, e.BundlePath)
 	if err != nil {
 		return err
 	}
-	h.s.containerGroup.Add(1)
 	h.s.containers[e.ID] = &containerInfo{
 		container: container,
 	}
 	ContainersCounter.Inc(1)
 	task := &StartTask{
 		Err:           e.Err,
-		IO:            io,
 		Container:     container,
-		Stdin:         e.Stdin,
-		Stdout:        e.Stdout,
-		Stderr:        e.Stderr,
 		StartResponse: e.StartResponse,
 	}
 	if e.Checkpoint != nil {

supervisor/delete.go

@@ -29,7 +29,6 @@ func (h *DeleteEvent) Handle(e *Event) error {
 			Pid:    e.Pid,
 		})
 		ContainersCounter.Dec(1)
-		h.s.containerGroup.Done()
 		ContainerDeleteTimer.UpdateSince(start)
 	}
 	return nil

supervisor/event.go

@@ -36,7 +36,9 @@ func NewEvent(t EventType) *Event {
 }
 
 type StartResponse struct {
-	Pid int
+	Stdin  string
+	Stdout string
+	Stderr string
 }
 
 type Event struct {
@@ -48,11 +50,12 @@ type Event struct {
 	Stderr        string
 	Stdin         string
 	Console       string
-	Pid           int
+	Pid           string
 	Status        int
 	Signal        os.Signal
-	Process       *specs.Process
+	Process       runtime.Process
 	State         runtime.State
+	ProcessSpec   *specs.Process
 	Containers    []runtime.Container
 	Checkpoint    *runtime.Checkpoint
 	Err           chan error

supervisor/exit.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/containerd/runtime"
 )
 
 type ExitEvent struct {
@@ -12,36 +13,36 @@ type ExitEvent struct {
 
 func (h *ExitEvent) Handle(e *Event) error {
 	start := time.Now()
-	logrus.WithFields(logrus.Fields{"pid": e.Pid, "status": e.Status}).
+	proc := e.Process
-		Debug("containerd: process exited")
+	status, err := proc.ExitStatus()
-	// is it the child process of a container
-	if info, ok := h.s.processes[e.Pid]; ok {
-		ne := NewEvent(ExecExitEventType)
-		ne.ID = info.container.ID()
-		ne.Pid = e.Pid
-		ne.Status = e.Status
-		h.s.SendEvent(ne)
-		return nil
-	}
-	// is it the main container's process
-	container, err := h.s.getContainerForPid(e.Pid)
 	if err != nil {
-		if err != errNoContainerForPid {
+		logrus.WithField("error", err).Error("containerd: get exit status")
-			logrus.WithField("error", err).Error("containerd: find containers main pid")
 	}
-		return nil
+	logrus.WithFields(logrus.Fields{"pid": proc.ID(), "status": status}).Debug("containerd: process exited")
-	}
+
-	container.SetExited(e.Status)
+	// if the process is the the init process of the container then
-	ne := NewEvent(DeleteEventType)
+	// fire a separate event for this process
-	ne.ID = container.ID()
+	if proc.ID() != runtime.InitProcessID {
-	ne.Pid = e.Pid
+		ne := NewEvent(ExecExitEventType)
-	ne.Status = e.Status
+		ne.ID = proc.Container().ID()
+		ne.Status = status
 		h.s.SendEvent(ne)
 
+		return nil
+	}
+	container := proc.Container()
+	ne := NewEvent(DeleteEventType)
+	ne.ID = container.ID()
+	ne.Status = status
+	ne.Pid = proc.ID()
+	h.s.SendEvent(ne)
+
+	// remove stats collection for container
 	stopCollect := NewEvent(StopStatsEventType)
 	stopCollect.ID = container.ID()
 	h.s.SendEvent(stopCollect)
 	ExitProcessTimer.UpdateSince(start)
+
 	return nil
 }
 
@@ -51,6 +52,7 @@ type ExecExitEvent struct {
 
 func (h *ExecExitEvent) Handle(e *Event) error {
 	// exec process: we remove this process without notifying the main event loop
+	/*
 		info := h.s.processes[e.Pid]
 			if err := info.container.RemoveProcess(e.Pid); err != nil {
 				logrus.WithField("error", err).Error("containerd: find container for pid")
@@ -60,5 +62,6 @@ func (h *ExecExitEvent) Handle(e *Event) error {
 			}
 			delete(h.s.processes, e.Pid)
 			h.s.notifySubscribers(e)
+	*/
 	return nil
 }

supervisor/machine.go

@@ -3,15 +3,12 @@ package supervisor
 import "github.com/cloudfoundry/gosigar"
 
 type Machine struct {
-	ID     string
 	Cpus   int
 	Memory int64
 }
 
-func CollectMachineInformation(id string) (Machine, error) {
+func CollectMachineInformation() (Machine, error) {
-	m := Machine{
+	m := Machine{}
-		ID: id,
-	}
 	cpu := sigar.CpuList{}
 	if err := cpu.Get(); err != nil {
 		return m, err

supervisor/monitor.go

@@ -0,0 +1,86 @@
+package supervisor
+
+import (
+	"sync"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/containerd/runtime"
+)
+
+func NewMonitor() (*Monitor, error) {
+	m := &Monitor{
+		processes: make(map[int]runtime.Process),
+		exits:     make(chan runtime.Process, 1024),
+	}
+	fd, err := syscall.EpollCreate1(0)
+	if err != nil {
+		return nil, err
+	}
+	m.epollFd = fd
+	go m.start()
+	return m, nil
+}
+
+type Monitor struct {
+	m         sync.Mutex
+	processes map[int]runtime.Process
+	exits     chan runtime.Process
+	epollFd   int
+}
+
+func (m *Monitor) Exits() chan runtime.Process {
+	return m.exits
+}
+
+func (m *Monitor) Monitor(p runtime.Process) error {
+	m.m.Lock()
+	defer m.m.Unlock()
+	fd := p.ExitFD()
+	event := syscall.EpollEvent{
+		Fd:     int32(fd),
+		Events: syscall.EPOLLHUP,
+	}
+	if err := syscall.EpollCtl(m.epollFd, syscall.EPOLL_CTL_ADD, fd, &event); err != nil {
+		return err
+	}
+	m.processes[fd] = p
+	return nil
+}
+
+func (m *Monitor) Close() error {
+	return syscall.Close(m.epollFd)
+}
+
+func (m *Monitor) start() {
+	var events [128]syscall.EpollEvent
+	for {
+		n, err := syscall.EpollWait(m.epollFd, events[:], -1)
+		if err != nil {
+			if err == syscall.EINTR {
+				continue
+			}
+			logrus.WithField("error", err).Fatal("containerd: epoll wait")
+		}
+		// process events
+		for i := 0; i < n; i++ {
+			if events[i].Events == syscall.EPOLLHUP {
+				fd := int(events[i].Fd)
+				m.m.Lock()
+				proc := m.processes[fd]
+				delete(m.processes, fd)
+				if err = syscall.EpollCtl(m.epollFd, syscall.EPOLL_CTL_DEL, fd, &syscall.EpollEvent{
+					Events: syscall.EPOLLHUP,
+					Fd:     int32(fd),
+				}); err != nil {
+					logrus.WithField("error", err).Fatal("containerd: epoll remove fd")
+				}
+				if err := proc.Close(); err != nil {
+					logrus.WithField("error", err).Error("containerd: close process IO")
+				}
+				m.m.Unlock()
+				m.exits <- proc
+			}
+		}
+	}
+}

supervisor/signal.go

@@ -5,6 +5,7 @@ type SignalEvent struct {
 }
 
 func (h *SignalEvent) Handle(e *Event) error {
+	/*
 		i, ok := h.s.containers[e.ID]
 		if !ok {
 			return ErrContainerNotFound
@@ -18,5 +19,6 @@ func (h *SignalEvent) Handle(e *Event) error {
 					return p.Signal(e.Signal)
 				}
 			}
+	*/
 	return ErrProcessNotFound
 }

supervisor/stats_collector.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/docker/containerd/api/grpc/types"
 	"github.com/docker/containerd/runtime"
 	"github.com/docker/docker/pkg/pubsub"
@@ -179,6 +178,7 @@ func (s *statsCollector) run() {
 			continue
 		}
 
+		/*
 			for _, pair := range pairs {
 					stats, err := pair.ct.Stats()
 					if err != nil {
@@ -188,6 +188,7 @@ func (s *statsCollector) run() {
 
 					pair.pub.Publish(convertToPb(stats))
 			}
+		*/
 	}
 }
 

supervisor/supervisor.go

@@ -1,18 +1,16 @@
 package supervisor
 
 import (
+	"io/ioutil"
 	"os"
-	"os/signal"
 	"path/filepath"
 	"sync"
-	"syscall"
 	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/containerd/chanotify"
 	"github.com/docker/containerd/eventloop"
 	"github.com/docker/containerd/runtime"
-	"github.com/opencontainers/runc/libcontainer"
 )
 
 const (
@@ -21,29 +19,27 @@ const (
 )
 
 // New returns an initialized Process supervisor.
-func New(id, stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error) {
+func New(stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error) {
 	if err := os.MkdirAll(stateDir, 0755); err != nil {
 		return nil, err
 	}
-	// register counters
+	machine, err := CollectMachineInformation()
-	r, err := newRuntime(filepath.Join(stateDir, id))
 	if err != nil {
 		return nil, err
 	}
-	machine, err := CollectMachineInformation(id)
+	monitor, err := NewMonitor()
 	if err != nil {
 		return nil, err
 	}
 	s := &Supervisor{
 		stateDir:       stateDir,
 		containers:     make(map[string]*containerInfo),
-		processes:      make(map[int]*containerInfo),
-		runtime:        r,
 		tasks:          tasks,
 		machine:        machine,
 		subscribers:    make(map[chan *Event]struct{}),
 		statsCollector: newStatsCollector(statsInterval),
 		el:             eventloop.NewChanLoop(defaultBufferSize),
+		monitor:        monitor,
 	}
 	if oom {
 		s.notifier = chanotify.New()
@@ -71,7 +67,10 @@ func New(id, stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, err
 		UnsubscribeStatsEventType: &UnsubscribeStatsEvent{s},
 		StopStatsEventType:        &StopStatsEvent{s},
 	}
-	// start the container workers for concurrent container starts
+	go s.exitHandler()
+	if err := s.restore(); err != nil {
+		return nil, err
+	}
 	return s, nil
 }
 
@@ -84,9 +83,7 @@ type Supervisor struct {
 	// stateDir is the directory on the system to store container runtime state information.
 	stateDir   string
 	containers map[string]*containerInfo
-	processes  map[int]*containerInfo
 	handlers   map[EventType]Handler
-	runtime    runtime.Runtime
 	events     chan *Event
 	tasks      chan *StartTask
 	// we need a lock around the subscribers map only because additions and deletions from
@@ -94,51 +91,18 @@ type Supervisor struct {
 	subscriberLock sync.RWMutex
 	subscribers    map[chan *Event]struct{}
 	machine        Machine
-	containerGroup sync.WaitGroup
 	statsCollector *statsCollector
 	notifier       *chanotify.Notifier
 	el             eventloop.EventLoop
+	monitor        *Monitor
 }
 
 // Stop closes all tasks and sends a SIGTERM to each container's pid1 then waits for they to
 // terminate.  After it has handled all the SIGCHILD events it will close the signals chan
 // and exit.  Stop is a non-blocking call and will return after the containers have been signaled
-func (s *Supervisor) Stop(sig chan os.Signal) {
+func (s *Supervisor) Stop() {
 	// Close the tasks channel so that no new containers get started
 	close(s.tasks)
-	// send a SIGTERM to all containers
-	for id, i := range s.containers {
-		c := i.container
-		logrus.WithField("id", id).Debug("sending TERM to container processes")
-		procs, err := c.Processes()
-		if err != nil {
-			logrus.WithField("id", id).Warn("get container processes")
-			continue
-		}
-		if len(procs) == 0 {
-			continue
-		}
-		mainProc := procs[0]
-		if err := mainProc.Signal(syscall.SIGTERM); err != nil {
-			pid, _ := mainProc.Pid()
-			logrus.WithFields(logrus.Fields{
-				"id":    id,
-				"pid":   pid,
-				"error": err,
-			}).Error("send SIGTERM to process")
-		}
-	}
-	go func() {
-		logrus.Debug("waiting for containers to exit")
-		s.containerGroup.Wait()
-		logrus.Debug("all containers exited")
-		if s.notifier != nil {
-			s.notifier.Close()
-		}
-		// stop receiving signals and close the channel
-		signal.Stop(sig)
-		close(sig)
-	}()
 }
 
 // Close closes any open files in the supervisor but expects that Stop has been
@@ -190,7 +154,6 @@ func (s *Supervisor) notifySubscribers(e *Event) {
 // state of the Supervisor
 func (s *Supervisor) Start() error {
 	logrus.WithFields(logrus.Fields{
-		"runtime":  s.runtime.Type(),
 		"stateDir": s.stateDir,
 	}).Debug("Supervisor started")
 	return s.el.Start()
@@ -202,45 +165,60 @@ func (s *Supervisor) Machine() Machine {
 	return s.machine
 }
 
-// getContainerForPid returns the container where the provided pid is the pid1 or main
-// process in the container
-func (s *Supervisor) getContainerForPid(pid int) (runtime.Container, error) {
-	for _, i := range s.containers {
-		container := i.container
-		cpid, err := container.Pid()
-		if err != nil {
-			if lerr, ok := err.(libcontainer.Error); ok {
-				if lerr.Code() == libcontainer.ProcessNotExecuted {
-					continue
-				}
-			}
-			logrus.WithField("error", err).Error("containerd: get container pid")
-		}
-		if pid == cpid {
-			return container, nil
-		}
-	}
-	return nil, errNoContainerForPid
-}
-
 // SendEvent sends the provided event the the supervisors main event loop
 func (s *Supervisor) SendEvent(evt *Event) {
 	EventsCounter.Inc(1)
 	s.el.Send(&commonEvent{data: evt, sv: s})
 }
 
-func (s *Supervisor) copyIO(stdin, stdout, stderr string, i *runtime.IO) (*copier, error) {
+func (s *Supervisor) exitHandler() {
-	config := &ioConfig{
+	for p := range s.monitor.Exits() {
-		Stdin:      i.Stdin,
+		e := NewEvent(ExitEventType)
-		Stdout:     i.Stdout,
+		e.Process = p
-		Stderr:     i.Stderr,
+		s.SendEvent(e)
-		StdoutPath: stdout,
-		StderrPath: stderr,
-		StdinPath:  stdin,
 	}
-	l, err := newCopier(config)
+}
-	if err != nil {
+
-		return nil, err
+func (s *Supervisor) monitorProcess(p runtime.Process) error {
-	}
+	return s.monitor.Monitor(p)
-	return l, nil
+}
+
+func (s *Supervisor) restore() error {
+	dirs, err := ioutil.ReadDir(s.stateDir)
+	if err != nil {
+		return err
+	}
+	for _, d := range dirs {
+		if !d.IsDir() {
+			continue
+		}
+		id := d.Name()
+		container, err := runtime.Load(s.stateDir, id)
+		if err != nil {
+			if err == runtime.ErrContainerExited {
+				logrus.WithField("id", id).Debug("containerd: container exited while away")
+				// TODO: fire events to do the removal
+				if err := os.RemoveAll(filepath.Join(s.stateDir, id)); err != nil {
+					logrus.WithField("error", err).Warn("containerd: remove container state")
+				}
+				continue
+			}
+			return err
+		}
+		processes, err := container.Processes()
+		if err != nil {
+			return err
+		}
+		ContainersCounter.Inc(1)
+		s.containers[id] = &containerInfo{
+			container: container,
+		}
+		logrus.WithField("id", id).Debug("containerd: container restored")
+		for _, p := range processes {
+			if err := s.monitorProcess(p); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
 }

supervisor/supervisor_linux.go

@@ -1,12 +0,0 @@
-// +build libcontainer
-
-package supervisor
-
-import (
-	"github.com/docker/containerd/linux"
-	"github.com/docker/containerd/runtime"
-)
-
-func newRuntime(stateDir string) (runtime.Runtime, error) {
-	return linux.NewRuntime(stateDir)
-}

supervisor/supervisor_runc.go

@@ -1,12 +0,0 @@
-// +build runc
-
-package supervisor
-
-import (
-	"github.com/docker/containerd/runc"
-	"github.com/docker/containerd/runtime"
-)
-
-func newRuntime(stateDir string) (runtime.Runtime, error) {
-	return runc.NewRuntime(stateDir)
-}

supervisor/supervisor_unsupported.go

@@ -1,13 +0,0 @@
-// +build !libcontainer,!runc
-
-package supervisor
-
-import (
-	"errors"
-
-	"github.com/docker/containerd/runtime"
-)
-
-func newRuntime(stateDir string) (runtime.Runtime, error) {
-	return nil, errors.New("unsupported platform")
-}

supervisor/update.go

@@ -27,6 +27,7 @@ func (h *UpdateEvent) Handle(e *Event) error {
 		}
 	}
 	if e.Signal != nil {
+		/*
 			// signal the pid1/main process of the container
 			processes, err := container.Processes()
 			if err != nil {
@@ -36,6 +37,7 @@ func (h *UpdateEvent) Handle(e *Event) error {
 				return ErrProcessNotFound
 			}
 			return processes[0].Signal(e.Signal)
+		*/
 	}
 	return nil
 }

supervisor/worker.go

@@ -38,17 +38,13 @@ type worker struct {
 func (w *worker) Start() {
 	defer w.wg.Done()
 	for t := range w.s.tasks {
-		started := time.Now()
+		var (
-		l, err := w.s.copyIO(t.Stdin, t.Stdout, t.Stderr, t.IO)
+			err     error
-		if err != nil {
+			process runtime.Process
-			evt := NewEvent(DeleteEventType)
+			started = time.Now()
-			evt.ID = t.Container.ID()
+		)
-			w.s.SendEvent(evt)
-			t.Err <- err
-			continue
-		}
-		w.s.containers[t.Container.ID()].copier = l
 		if t.Checkpoint != "" {
+			/*
 				if err := t.Container.Restore(t.Checkpoint); err != nil {
 					evt := NewEvent(DeleteEventType)
 					evt.ID = t.Container.ID()
@@ -56,8 +52,10 @@ func (w *worker) Start() {
 					t.Err <- err
 					continue
 				}
+			*/
 		} else {
-			if err := t.Container.Start(); err != nil {
+			process, err = t.Container.Start()
+			if err != nil {
 				evt := NewEvent(DeleteEventType)
 				evt.ID = t.Container.ID()
 				w.s.SendEvent(evt)
@@ -65,22 +63,25 @@ func (w *worker) Start() {
 				continue
 			}
 		}
-		pid, err := t.Container.Pid()
+		/*
-		if err != nil {
-			logrus.WithField("error", err).Error("containerd: get container main pid")
-		}
 		   if w.s.notifier != nil {
 		       n, err := t.Container.OOM()
 		       if err != nil {
 		           logrus.WithField("error", err).Error("containerd: notify OOM events")
 		       } else {
-				w.s.notifier.Add(t.Container.ID(), n)
+		           w.s.notifier.Add(n, t.Container.ID())
 		       }
 		   }
+		*/
+		if err := w.s.monitorProcess(process); err != nil {
+			logrus.WithField("error", err).Error("containerd: add process to monitor")
+		}
 		ContainerStartTimer.UpdateSince(started)
 		t.Err <- nil
 		t.StartResponse <- StartResponse{
-			Pid: pid,
+			Stdin:  process.Stdin(),
+			Stdout: process.Stdout(),
+			Stderr: process.Stderr(),
 		}
 	}
 }

util/reaper.go

@@ -0,0 +1,38 @@
+package util
+
+import (
+	"syscall"
+
+	"github.com/opencontainers/runc/libcontainer/utils"
+)
+
+// Exit is the wait4 information from an exited process
+type Exit struct {
+	Pid    int
+	Status int
+}
+
+// Reap reaps all child processes for the calling process and returns their
+// exit information
+func Reap() (exits []Exit, err error) {
+	var (
+		ws  syscall.WaitStatus
+		rus syscall.Rusage
+	)
+	for {
+		pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, &rus)
+		if err != nil {
+			if err == syscall.ECHILD {
+				return exits, nil
+			}
+			return exits, err
+		}
+		if pid <= 0 {
+			return exits, nil
+		}
+		exits = append(exits, Exit{
+			Pid:    pid,
+			Status: utils.ExitStatus(ws),
+		})
+	}
+}