vbatts
/
containerd
1
0
Fork 0
Code Pull Requests Releases Activity

Add shim for reattach of processes 

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Remove runtime files from containerd

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Update supervisor for orphaned containers

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Remove ctr/container.go back to rpc calls

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add attach to loaded container

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add monitor based on epoll for process exits

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Convert pids in containerd to string

This is so that we no longer care about linux or system level pids and
processes in containerd have user defined process id(pid) kinda like the
exec process ids that docker has today.

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add reaper back to containerd

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Implement list containers with new process model

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Implement restore of processes

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add NONBLOCK to exit fifo open

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Implement tty reattach

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Fix race in exit pipe creation

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add delete to shim

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Update shim to use pid-file and not stdout

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
This commit is contained in:
Michael Crosby 2016-01-06 13:32:46 -08:00
parent 8d1f71c3d7
commit fe38efda50
33 changed files with 1210 additions and 1836 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
containerd/containerd
containerd-shim/containerd-shim
bin/
ctr/ctr
hack/benchmark

7
Makefile
View File

@ -1,4 +1,4 @@
BUILDTAGS=libcontainer
BUILDTAGS=
# if this session isn't interactive, then we don't want to allocate a
# TTY, which would fail, but if it is interactive, we do want to attach
@ -13,7 +13,7 @@ DOCKER_RUN := docker run --rm -i $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
export GOPATH:=$(CURDIR)/vendor:$(GOPATH)
all: client daemon
all: client daemon shim
bin:
mkdir -p bin/
@ -27,6 +27,9 @@ client: bin
daemon: bin
cd containerd && go build -tags "$(BUILDTAGS)" -o ../bin/containerd
shim: bin
cd containerd-shim && go build -tags "$(BUILDTAGS)" -o ../bin/containerd-shim
dbuild:
@docker build --rm --force-rm -t "$(DOCKER_IMAGE)" .

53
api/grpc/server/server.go
View File

@ -33,10 +33,6 @@ func (s *apiServer) CreateContainer(ctx context.Context, c *types.CreateContaine
e := supervisor.NewEvent(supervisor.StartContainerEventType)
e.ID = c.Id
e.BundlePath = c.BundlePath
e.Stdout = c.Stdout
e.Stderr = c.Stderr
e.Stdin = c.Stdin
e.Console = c.Console
e.StartResponse = make(chan supervisor.StartResponse, 1)
if c.Checkpoint != "" {
e.Checkpoint = &runtime.Checkpoint{
@ -49,14 +45,16 @@ func (s *apiServer) CreateContainer(ctx context.Context, c *types.CreateContaine
}
sr := <-e.StartResponse
return &types.CreateContainerResponse{
Pid: uint32(sr.Pid),
Stdin: sr.Stdin,
Stdout: sr.Stdout,
Stderr: sr.Stderr,
}, nil
}
func (s *apiServer) Signal(ctx context.Context, r *types.SignalRequest) (*types.SignalResponse, error) {
e := supervisor.NewEvent(supervisor.SignalEventType)
e.ID = r.Id
e.Pid = int(r.Pid)
e.Pid = r.Pid
e.Signal = syscall.Signal(int(r.Signal))
s.sv.SendEvent(e)
if err := <-e.Err; err != nil {
@ -79,7 +77,7 @@ func (s *apiServer) AddProcess(ctx context.Context, r *types.AddProcessRequest)
}
e := supervisor.NewEvent(supervisor.AddProcessEventType)
e.ID = r.Id
e.Process = process
e.ProcessSpec = process
e.Console = r.Console
e.Stdin = r.Stdin
e.Stdout = r.Stdout
@ -88,7 +86,7 @@ func (s *apiServer) AddProcess(ctx context.Context, r *types.AddProcessRequest)
if err := <-e.Err; err != nil {
return nil, err
}
return &types.AddProcessResponse{Pid: uint32(e.Pid)}, nil
return &types.AddProcessResponse{}, nil
}
func (s *apiServer) CreateCheckpoint(ctx context.Context, r *types.CreateCheckpointRequest) (*types.CreateCheckpointResponse, error) {
@ -140,21 +138,23 @@ func (s *apiServer) ListCheckpoint(ctx context.Context, r *types.ListCheckpointR
if container == nil {
return nil, grpc.Errorf(codes.NotFound, "no such containers")
}
checkpoints, err := container.Checkpoints()
if err != nil {
return nil, err
}
var out []*types.Checkpoint
for _, c := range checkpoints {
out = append(out, &types.Checkpoint{
Name: c.Name,
Tcp: c.Tcp,
Shell: c.Shell,
UnixSockets: c.UnixSockets,
// TODO: figure out timestamp
//Timestamp: c.Timestamp,
})
}
/*
checkpoints, err := container.Checkpoints()
if err != nil {
return nil, err
}
for _, c := range checkpoints {
out = append(out, &types.Checkpoint{
Name: c.Name,
Tcp: c.Tcp,
Shell: c.Shell,
UnixSockets: c.UnixSockets,
// TODO: figure out timestamp
//Timestamp: c.Timestamp,
})
}
*/
return &types.ListCheckpointResponse{Checkpoints: out}, nil
}
@ -167,7 +167,6 @@ func (s *apiServer) State(ctx context.Context, r *types.StateRequest) (*types.St
m := s.sv.Machine()
state := &types.StateResponse{
Machine: &types.Machine{
Id: m.ID,
Cpus: uint32(m.Cpus),
Memory: uint64(m.Cpus),
},
@ -179,13 +178,9 @@ func (s *apiServer) State(ctx context.Context, r *types.StateRequest) (*types.St
}
var procs []*types.Process
for _, p := range processes {
pid, err := p.Pid()
if err != nil {
logrus.WithField("error", err).Error("get process pid")
}
oldProc := p.Spec()
procs = append(procs, &types.Process{
Pid: uint32(pid),
Pid: p.ID(),
Terminal: oldProc.Terminal,
Args: oldProc.Args,
Env: oldProc.Env,
@ -231,7 +226,7 @@ func (s *apiServer) Events(r *types.EventsRequest, stream types.API_EventsServer
ev = &types.Event{
Type: "exit",
Id: evt.ID,
Pid: uint32(evt.Pid),
Pid: evt.Pid,
Status: uint32(evt.Status),
}
case supervisor.OOMEventType:

205
api/grpc/types/api.pb.go
View File

@ -65,11 +65,7 @@ var _ = math.Inf
type CreateContainerRequest struct {
Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
BundlePath string `protobuf:"bytes,2,opt,name=bundlePath" json:"bundlePath,omitempty"`
Stdin string `protobuf:"bytes,3,opt,name=stdin" json:"stdin,omitempty"`
Stdout string `protobuf:"bytes,4,opt,name=stdout" json:"stdout,omitempty"`
Stderr string `protobuf:"bytes,5,opt,name=stderr" json:"stderr,omitempty"`
Console string `protobuf:"bytes,6,opt,name=console" json:"console,omitempty"`
Checkpoint string `protobuf:"bytes,7,opt,name=checkpoint" json:"checkpoint,omitempty"`
Checkpoint string `protobuf:"bytes,3,opt,name=checkpoint" json:"checkpoint,omitempty"`
}
func (m *CreateContainerRequest) Reset() { *m = CreateContainerRequest{} }
@ -78,7 +74,9 @@ func (*CreateContainerRequest) ProtoMessage() {}
func (*CreateContainerRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
type CreateContainerResponse struct {
Pid uint32 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"`
Stdin string `protobuf:"bytes,1,opt,name=stdin" json:"stdin,omitempty"`
Stdout string `protobuf:"bytes,2,opt,name=stdout" json:"stdout,omitempty"`
Stderr string `protobuf:"bytes,3,opt,name=stderr" json:"stderr,omitempty"`
}
func (m *CreateContainerResponse) Reset() { *m = CreateContainerResponse{} }
@ -88,7 +86,7 @@ func (*CreateContainerResponse) Descriptor() ([]byte, []int) { return fileDescri
type SignalRequest struct {
Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
Pid uint32 `protobuf:"varint,2,opt,name=pid" json:"pid,omitempty"`
Pid string `protobuf:"bytes,2,opt,name=pid" json:"pid,omitempty"`
Signal uint32 `protobuf:"varint,3,opt,name=signal" json:"signal,omitempty"`
}
@ -142,7 +140,6 @@ func (*User) ProtoMessage() {}
func (*User) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
type AddProcessResponse struct {
Pid uint32 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"`
}
func (m *AddProcessResponse) Reset() { *m = AddProcessResponse{} }
@ -249,7 +246,7 @@ func (*ContainerState) ProtoMessage() {}
func (*ContainerState) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
type Process struct {
Pid uint32 `protobuf:"varint,1,opt,name=pid" json:"pid,omitempty"`
Pid string `protobuf:"bytes,1,opt,name=pid" json:"pid,omitempty"`
Terminal bool `protobuf:"varint,2,opt,name=terminal" json:"terminal,omitempty"`
User *User `protobuf:"bytes,3,opt,name=user" json:"user,omitempty"`
Args []string `protobuf:"bytes,4,rep,name=args" json:"args,omitempty"`
@ -291,9 +288,8 @@ func (m *Container) GetProcesses() []*Process {
// Machine is information about machine on which containerd is run
type Machine struct {
Id string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
Cpus uint32 `protobuf:"varint,2,opt,name=cpus" json:"cpus,omitempty"`
Memory uint64 `protobuf:"varint,3,opt,name=memory" json:"memory,omitempty"`
Cpus uint32 `protobuf:"varint,1,opt,name=cpus" json:"cpus,omitempty"`
Memory uint64 `protobuf:"varint,2,opt,name=memory" json:"memory,omitempty"`
}
func (m *Machine) Reset() { *m = Machine{} }
@ -358,7 +354,7 @@ type Event struct {
Id string `protobuf:"bytes,2,opt,name=id" json:"id,omitempty"`
Status uint32 `protobuf:"varint,3,opt,name=status" json:"status,omitempty"`
BundlePath string `protobuf:"bytes,4,opt,name=bundlePath" json:"bundlePath,omitempty"`
Pid uint32 `protobuf:"varint,5,opt,name=pid" json:"pid,omitempty"`
Pid string `protobuf:"bytes,5,opt,name=pid" json:"pid,omitempty"`
Signal uint32 `protobuf:"varint,7,opt,name=signal" json:"signal,omitempty"`
Process *Process `protobuf:"bytes,8,opt,name=process" json:"process,omitempty"`
Containers []*Container `protobuf:"bytes,9,rep,name=containers" json:"containers,omitempty"`
@ -1087,96 +1083,95 @@ var _API_serviceDesc = grpc.ServiceDesc{
}
var fileDescriptor0 = []byte{
// 1454 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x6e, 0x1c, 0x45,
0x17, 0xf6, 0xcc, 0xf4, 0x6c, 0x67, 0x16, 0xdb, 0xed, 0xd8, 0x19, 0xcf, 0xff, 0x87, 0x24, 0x4d,
0x08, 0x11, 0x8a, 0xac, 0xe0, 0xb0, 0x84, 0x70, 0x01, 0xc1, 0x89, 0x12, 0x50, 0x02, 0x56, 0x62,
0x23, 0x71, 0xc3, 0xa8, 0xdd, 0x5d, 0x8c, 0x8b, 0xe9, 0x8d, 0xee, 0x6a, 0x2f, 0xaf, 0x80, 0x78,
0x1c, 0xc4, 0x03, 0x20, 0x71, 0xcf, 0x73, 0xf0, 0x14, 0x9c, 0x5a, 0xba, 0x7a, 0x99, 0xc5, 0x70,
0xc1, 0xcd, 0x48, 0x55, 0x75, 0xce, 0x77, 0xce, 0xf9, 0xce, 0xd2, 0x55, 0x03, 0x5d, 0x3b, 0xa2,
0x7b, 0x51, 0x1c, 0xb2, 0xd0, 0x6c, 0xb2, 0xcb, 0x88, 0x24, 0xd6, 0x2f, 0x35, 0xd8, 0x39, 0x88,
0x89, 0xcd, 0xc8, 0x41, 0x18, 0x30, 0x9b, 0x06, 0x24, 0x7e, 0x4d, 0x7e, 0x4a, 0x49, 0xc2, 0x4c,
0x80, 0x3a, 0x75, 0x47, 0xb5, 0x5b, 0xb5, 0x7b, 0x5d, 0x13, 0x17, 0x27, 0x69, 0xe0, 0x7a, 0xe4,
0xd0, 0x66, 0xa7, 0xa3, 0xba, 0xd8, 0x1b, 0x40, 0x33, 0x61, 0x2e, 0x0d, 0x46, 0x0d, 0xb1, 0x1c,
0x42, 0x0b, 0x97, 0x61, 0xca, 0x46, 0x46, 0x61, 0x4d, 0xe2, 0x78, 0xd4, 0x14, 0xeb, 0x75, 0x68,
0x3b, 0x61, 0x90, 0x84, 0x1e, 0x19, 0xb5, 0x32, 0x4c, 0xe7, 0x94, 0x38, 0xb3, 0x28, 0xa4, 0x01,
0x1b, 0xb5, 0xf9, 0x9e, 0x75, 0x17, 0xae, 0xcf, 0x79, 0x93, 0x44, 0xa8, 0x46, 0xcc, 0x1e, 0x34,
0x22, 0xe5, 0xcf, 0xc0, 0x7a, 0x04, 0x83, 0x37, 0x74, 0x1a, 0xd8, 0xde, 0x22, 0x67, 0x95, 0x24,
0xf7, 0x72, 0x20, 0xdc, 0x10, 0x92, 0xc2, 0xcd, 0x81, 0xb5, 0x01, 0xc3, 0x4c, 0x53, 0x02, 0x5b,
0xbf, 0xd6, 0x60, 0xf3, 0x89, 0xeb, 0x1e, 0xc6, 0xa1, 0x43, 0x92, 0x64, 0x11, 0xe0, 0x06, 0x74,
0x18, 0x89, 0x7d, 0xca, 0x51, 0x38, 0x6a, 0xc7, 0xdc, 0x05, 0x23, 0x4d, 0x48, 0x2c, 0x30, 0x7b,
0xfb, 0xbd, 0x3d, 0x41, 0xe6, 0xde, 0x31, 0x6e, 0x99, 0x7d, 0x30, 0xec, 0x78, 0x9a, 0x20, 0x0b,
0x0d, 0xe9, 0x0b, 0x09, 0xce, 0x90, 0x02, 0xb5, 0x70, 0xce, 0x5d, 0x15, 0xbe, 0xa6, 0xaf, 0x5d,
0xa1, 0xaf, 0x53, 0xa1, 0xaf, 0x5b, 0xa5, 0x0f, 0x04, 0x55, 0x8f, 0xc0, 0x10, 0xf6, 0x10, 0x34,
0xcd, 0x78, 0xe1, 0x8b, 0xa9, 0x0e, 0x7d, 0x07, 0x86, 0xb6, 0xeb, 0x52, 0x46, 0x43, 0x74, 0xfc,
0x39, 0x75, 0x13, 0x74, 0xb7, 0x81, 0x14, 0xdc, 0x06, 0xb3, 0x18, 0xef, 0x22, 0x7e, 0x5f, 0xea,
0x3c, 0xe8, 0x0c, 0x2d, 0x22, 0xe6, 0x9d, 0x52, 0x0a, 0xeb, 0x82, 0x8c, 0x4d, 0x45, 0x46, 0xae,
0x69, 0x8d, 0x61, 0x34, 0x8f, 0xa6, 0xd8, 0x7f, 0x08, 0xd7, 0x9f, 0x12, 0x8f, 0x5c, 0x65, 0x09,
0x59, 0x0d, 0x6c, 0x9f, 0xc8, 0xd2, 0xe3, 0x80, 0xf3, 0x4a, 0x0a, 0xf0, 0x6d, 0xd8, 0x7e, 0x49,
0x13, 0xb6, 0x12, 0xce, 0xfa, 0x0e, 0x20, 0x17, 0xd0, 0xe0, 0xda, 0x14, 0xb9, 0xa0, 0x4c, 0x65,
0x1a, 0x69, 0x61, 0x4e, 0x24, 0x12, 0xdd, 0x31, 0xb7, 0xa0, 0x97, 0x06, 0xf4, 0xe2, 0x4d, 0xe8,
0xcc, 0x08, 0x4b, 0x44, 0xa1, 0x77, 0x44, 0x22, 0x4f, 0x89, 0xe7, 0x89, 0x3a, 0xef, 0x58, 0x9f,
0xc3, 0x4e, 0xd5, 0xbe, 0x62, 0xf8, 0x2e, 0xf4, 0x72, 0xb6, 0x12, 0xb4, 0xd6, 0x58, 0x4c, 0xd7,
0x10, 0xfa, 0x6f, 0x18, 0xb2, 0xa5, 0x1c, 0xb7, 0x6e, 0xc1, 0x50, 0xb7, 0x83, 0x38, 0x90, 0xc5,
0x61, 0xb3, 0x34, 0x51, 0xe1, 0xcc, 0xa0, 0xad, 0xd2, 0x59, 0x4a, 0xe3, 0x7f, 0x52, 0xb8, 0x96,
0x07, 0x5d, 0xed, 0xce, 0xf2, 0x1c, 0x55, 0x46, 0x86, 0x9c, 0x11, 0xb7, 0xa1, 0x1b, 0x49, 0x3f,
0x89, 0xb4, 0xd3, 0xdb, 0x1f, 0x2a, 0x17, 0x32, 0xff, 0xf3, 0xd0, 0xc4, 0xd8, 0xc0, 0xfa, 0x68,
0xbf, 0xb2, 0x9d, 0x53, 0x34, 0x56, 0xb5, 0xe5, 0x44, 0x28, 0xa4, 0x9b, 0xdc, 0x27, 0x7e, 0x18,
0x5f, 0x0a, 0x3b, 0x86, 0xf5, 0x2d, 0x8e, 0x07, 0xc9, 0xa0, 0xa2, 0xfe, 0x0e, 0x16, 0x6a, 0xe6,
0x73, 0xc6, 0xfc, 0x46, 0xc6, 0xbc, 0x0e, 0xe6, 0x26, 0xb4, 0x7d, 0x69, 0x4b, 0xd5, 0x72, 0xe6,
0x9c, 0xf2, 0xc0, 0x7a, 0x0a, 0x3b, 0xc7, 0x91, 0x7b, 0xd5, 0xb0, 0xcc, 0x47, 0x4e, 0x3e, 0x82,
0x64, 0x48, 0x82, 0x05, 0x6b, 0x17, 0xae, 0xcf, 0xa1, 0xa8, 0xe2, 0x5d, 0x87, 0xc1, 0xb3, 0x33,
0x82, 0xd5, 0x91, 0xe5, 0xfe, 0xcf, 0x1a, 0x34, 0xc5, 0x0e, 0x8f, 0x98, 0x3b, 0xa3, 0x6c, 0x48,
0x7b, 0xf5, 0x7c, 0x54, 0x68, 0xfc, 0x41, 0x85, 0x79, 0xa3, 0x38, 0x13, 0x9b, 0x95, 0x99, 0xd8,
0x16, 0x6b, 0x8c, 0x5b, 0xa5, 0x45, 0x0c, 0x9f, 0xf9, 0xa4, 0x94, 0xe9, 0xeb, 0x2e, 0xa1, 0xaf,
0x3c, 0x0d, 0x60, 0xd9, 0x34, 0xf8, 0xad, 0x06, 0xfd, 0xaf, 0x09, 0x3b, 0x0f, 0xe3, 0x19, 0x4f,
0x52, 0x52, 0x69, 0x3f, 0xac, 0xd9, 0xf8, 0x62, 0x72, 0x72, 0xc9, 0x88, 0xcc, 0xae, 0xc1, 0xe3,
0xc1, 0x9d, 0x43, 0x5b, 0x36, 0x9d, 0xc8, 0xb0, 0xb9, 0x09, 0xdd, 0xd7, 0x17, 0x13, 0x1c, 0x8f,
0x61, 0x2c, 0xfb, 0x50, 0x88, 0xe1, 0x96, 0x1b, 0x87, 0x51, 0x44, 0x64, 0xa4, 0x06, 0x07, 0x3b,
0xca, 0xc0, 0x5a, 0x99, 0x14, 0xee, 0x44, 0x0a, 0xac, 0x9d, 0x81, 0x1d, 0x69, 0xb0, 0x4e, 0x41,
0x2c, 0x03, 0xeb, 0x8a, 0xaa, 0xf2, 0xa1, 0x73, 0x10, 0xa5, 0xc7, 0x89, 0x3d, 0x25, 0x7c, 0x12,
0xb0, 0x90, 0xd9, 0xde, 0x24, 0xe5, 0x4b, 0xe1, 0xba, 0x61, 0x5e, 0x83, 0x7e, 0x44, 0x62, 0xac,
0x4b, 0xb5, 0x5b, 0x47, 0xa2, 0x0c, 0xf3, 0x7f, 0xb0, 0x25, 0x96, 0x13, 0x1a, 0x4c, 0x66, 0x24,
0x0e, 0x88, 0xe7, 0x87, 0x2e, 0x51, 0x71, 0xec, 0xc2, 0xa6, 0x3e, 0xe4, 0x8d, 0x29, 0x8e, 0x44,
0x3c, 0xd6, 0x11, 0x0c, 0x8f, 0x4e, 0xf1, 0x5b, 0xcd, 0x3c, 0x1a, 0x4c, 0x9f, 0xda, 0xcc, 0xe6,
0xdf, 0x00, 0xc4, 0xa7, 0xa1, 0x9b, 0x28, 0x83, 0xa8, 0xcd, 0xa4, 0x08, 0x71, 0x27, 0xd9, 0x91,
0x24, 0x0d, 0x87, 0x7f, 0x7e, 0xc4, 0xa8, 0xaf, 0x0c, 0x5a, 0xdf, 0x8b, 0x20, 0x24, 0xf1, 0x16,
0x74, 0x73, 0x67, 0x6b, 0x22, 0x5f, 0xeb, 0x59, 0xbe, 0xb2, 0x40, 0xf7, 0x60, 0x9d, 0x69, 0x2f,
0x26, 0x58, 0xb5, 0xb6, 0xea, 0x8d, 0x6d, 0x25, 0x59, 0xf6, 0xd1, 0xfa, 0x0c, 0xe0, 0x95, 0x68,
0x45, 0xe1, 0x31, 0xce, 0xc6, 0x22, 0x41, 0x48, 0xb4, 0x6f, 0x5f, 0x68, 0x76, 0xf8, 0x16, 0xc6,
0xf4, 0x83, 0x4d, 0x3d, 0x07, 0x2b, 0x46, 0x3a, 0xf8, 0x57, 0x0d, 0x7a, 0x12, 0x41, 0x3a, 0x89,
0x10, 0x0e, 0xb6, 0x5f, 0x06, 0x71, 0x2b, 0x43, 0x2c, 0x7f, 0x6d, 0x0a, 0x36, 0xb1, 0x0c, 0x93,
0x73, 0x3b, 0x52, 0x56, 0x1a, 0xcb, 0xc4, 0xde, 0x85, 0xbe, 0xcc, 0x86, 0x12, 0x34, 0x96, 0x09,
0xde, 0xe7, 0x1f, 0x6a, 0xf4, 0x44, 0xcc, 0xc2, 0xde, 0xfe, 0x8d, 0x92, 0x84, 0xf0, 0x71, 0x4f,
0xfc, 0x3e, 0x0b, 0x58, 0x7c, 0x39, 0xbe, 0x0f, 0x90, 0xaf, 0x78, 0xdb, 0xcd, 0xc8, 0xa5, 0xaa,
0x6c, 0x8c, 0xe4, 0xcc, 0xf6, 0x52, 0x15, 0xf9, 0xe3, 0xfa, 0xa3, 0x9a, 0xf5, 0x15, 0xac, 0x7f,
0xe1, 0xcd, 0x68, 0x58, 0x50, 0x41, 0x29, 0xdf, 0xfe, 0x31, 0x8c, 0x55, 0xbc, 0x7c, 0x49, 0x03,
0x5c, 0x4a, 0xba, 0xb0, 0xef, 0xc3, 0x48, 0x4d, 0x53, 0x8d, 0x27, 0xeb, 0xe5, 0xf7, 0x06, 0x40,
0x0e, 0x66, 0x3e, 0x86, 0x31, 0x0d, 0x27, 0x58, 0x52, 0x67, 0xd4, 0x21, 0xb2, 0x05, 0x26, 0x31,
0x71, 0xd2, 0x38, 0xa1, 0x67, 0x44, 0x8d, 0xc0, 0x1d, 0x15, 0x4b, 0xd5, 0x87, 0x0f, 0x61, 0x3b,
0xd7, 0x75, 0x0b, 0x6a, 0xf5, 0x95, 0x6a, 0x0f, 0x61, 0x0b, 0xd5, 0x70, 0x70, 0xa5, 0x25, 0xa5,
0xc6, 0x4a, 0xa5, 0x4f, 0x60, 0xb7, 0xe0, 0x27, 0xaf, 0xd4, 0x82, 0xaa, 0xb1, 0x52, 0xf5, 0x23,
0xd8, 0x41, 0xd5, 0x73, 0x9b, 0xb2, 0xaa, 0x5e, 0xf3, 0x1f, 0xf8, 0xe9, 0x93, 0x78, 0x5a, 0xf2,
0xb3, 0xb5, 0x52, 0xe9, 0x7d, 0xd8, 0x44, 0xa5, 0x8a, 0x9d, 0xf6, 0x55, 0x2a, 0x09, 0x71, 0x18,
0x4e, 0x95, 0x82, 0x4a, 0x67, 0x95, 0x8a, 0xf5, 0x04, 0xfa, 0x2f, 0xd2, 0x29, 0x61, 0xde, 0x89,
0xae, 0xfe, 0x7f, 0xdb, 0x40, 0x3f, 0xd7, 0xa1, 0x77, 0x30, 0x8d, 0xc3, 0x34, 0x2a, 0x75, 0xb9,
0xac, 0xe1, 0xb9, 0x2e, 0x97, 0x32, 0xf7, 0xa0, 0x2f, 0x3f, 0xa0, 0x4a, 0x4c, 0x36, 0x97, 0x39,
0x5f, 0xea, 0xfc, 0x12, 0x73, 0xc2, 0x7d, 0x56, 0x82, 0xe5, 0xf6, 0x2a, 0x94, 0xdf, 0xa7, 0x30,
0x38, 0x95, 0x81, 0x28, 0x49, 0x99, 0xca, 0x3b, 0x99, 0xe5, 0xdc, 0xc1, 0xbd, 0x62, 0xc0, 0xb2,
0x89, 0x5e, 0xc0, 0xe6, 0xdc, 0x66, 0xb9, 0x97, 0xac, 0x62, 0x2f, 0xf5, 0xf6, 0xb7, 0x14, 0x6c,
0x51, 0x4b, 0x34, 0x58, 0x04, 0x4d, 0xe9, 0xcf, 0x7b, 0x30, 0x08, 0xe4, 0x47, 0x47, 0x33, 0xd1,
0x28, 0x28, 0x96, 0x3e, 0x48, 0xc8, 0x86, 0x23, 0xfc, 0x5b, 0xc8, 0x46, 0x91, 0x5b, 0xcc, 0x07,
0xaf, 0x08, 0x14, 0xf3, 0x23, 0x45, 0xff, 0x58, 0xde, 0xde, 0x16, 0x3d, 0x24, 0xf6, 0xff, 0x68,
0x42, 0xe3, 0xc9, 0xe1, 0x97, 0xe6, 0x6b, 0x58, 0xaf, 0x3c, 0x73, 0xcc, 0x6c, 0xac, 0x2c, 0x7e,
0x8c, 0x8d, 0xdf, 0x5a, 0x76, 0xac, 0x2e, 0x0e, 0x6b, 0x1c, 0xb3, 0x72, 0xab, 0xd0, 0x98, 0x8b,
0xef, 0x2c, 0x1a, 0x73, 0xd9, 0x65, 0x64, 0xcd, 0xfc, 0x18, 0x5a, 0xf2, 0xb1, 0x64, 0x5e, 0x53,
0xb2, 0xa5, 0x57, 0xd7, 0x78, 0xbb, 0xb2, 0xab, 0x15, 0x0f, 0x00, 0xf2, 0x27, 0x86, 0x39, 0x52,
0x62, 0x73, 0xaf, 0xac, 0xf1, 0xee, 0x82, 0x13, 0x0d, 0x72, 0x0c, 0x1b, 0xd5, 0x67, 0x83, 0x59,
0xe1, 0xa1, 0x7a, 0xc9, 0x1f, 0xdf, 0x5c, 0x7a, 0x5e, 0x84, 0xad, 0x3e, 0x1e, 0x34, 0xec, 0x92,
0xa7, 0x88, 0x86, 0x5d, 0xfa, 0xea, 0x58, 0x33, 0xbf, 0x81, 0x61, 0xf9, 0xde, 0x6f, 0xfe, 0x5f,
0x29, 0x2d, 0x7c, 0x8e, 0x8c, 0x6f, 0x2c, 0x39, 0xd5, 0x80, 0x1f, 0xc8, 0xd2, 0xc5, 0xbb, 0x46,
0xc6, 0x72, 0xe1, 0x51, 0x30, 0xbe, 0x56, 0xde, 0xd4, 0x5a, 0x0f, 0xa0, 0x25, 0x6f, 0x90, 0x3a,
0x65, 0xa5, 0x0b, 0xe5, 0xb8, 0x5f, 0xdc, 0xb5, 0xd6, 0x1e, 0xd4, 0x70, 0x4a, 0x75, 0x9e, 0x13,
0x26, 0xeb, 0xb9, 0x68, 0x6a, 0x4e, 0x45, 0x6c, 0x72, 0x95, 0x93, 0x96, 0xf8, 0x0f, 0xe1, 0xe1,
0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xa6, 0xf1, 0xd0, 0x3c, 0x50, 0x10, 0x00, 0x00,
// 1440 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x72, 0xdc, 0x44,
0x17, 0xf6, 0xec, 0x33, 0x67, 0x16, 0xdb, 0xf2, 0x92, 0xf1, 0xfc, 0x7f, 0x88, 0x11, 0x81, 0xa4,
0xa8, 0x94, 0x2b, 0x38, 0x2c, 0x21, 0x5c, 0x40, 0x70, 0x52, 0x09, 0x54, 0x02, 0xae, 0xc4, 0xa6,
0x8a, 0x1b, 0xa6, 0x64, 0xa9, 0x19, 0x8b, 0xd1, 0x48, 0x42, 0x6a, 0x79, 0x79, 0x05, 0x9e, 0x87,
0xe2, 0x01, 0xa8, 0xe2, 0x9e, 0xe7, 0xe0, 0x29, 0x38, 0xdd, 0x7d, 0xd4, 0x5a, 0x66, 0x09, 0x5c,
0x70, 0x33, 0x55, 0xdd, 0x7d, 0xce, 0x77, 0xbe, 0xb3, 0xaa, 0x7b, 0xa0, 0x63, 0x85, 0xee, 0x41,
0x18, 0x05, 0x3c, 0x30, 0x1a, 0xfc, 0x3a, 0x64, 0xb1, 0xf9, 0x02, 0x76, 0x8f, 0x22, 0x66, 0x71,
0x76, 0x14, 0xf8, 0xdc, 0x72, 0x7d, 0x16, 0xbd, 0x62, 0x3f, 0x27, 0x2c, 0xe6, 0x06, 0x40, 0xd5,
0x75, 0x86, 0x95, 0xfd, 0xca, 0xdd, 0x8e, 0x81, 0x8b, 0xb3, 0xc4, 0x77, 0x3c, 0x76, 0x6c, 0xf1,
0xf3, 0x61, 0x35, 0xdd, 0xb3, 0xcf, 0x99, 0x3d, 0x0d, 0x03, 0xd7, 0xe7, 0xc3, 0x9a, 0xd8, 0x33,
0x9f, 0xc3, 0x8d, 0x39, 0xb4, 0x38, 0x0c, 0xfc, 0x98, 0x19, 0x7d, 0x68, 0xc4, 0xdc, 0x71, 0x7d,
0x42, 0x1c, 0x40, 0x13, 0x97, 0x41, 0xc2, 0x09, 0x4d, 0xad, 0x59, 0x14, 0x11, 0xd2, 0x43, 0xe8,
0xbf, 0x76, 0x27, 0xbe, 0xe5, 0x2d, 0xa2, 0xd3, 0x85, 0x5a, 0x88, 0x8b, 0x4c, 0x53, 0x4a, 0x4a,
0xcd, 0xbe, 0xb9, 0x01, 0x83, 0x54, 0x53, 0x99, 0x36, 0x7f, 0xad, 0xc0, 0xe6, 0x63, 0xc7, 0x39,
0x8e, 0x02, 0x9b, 0xc5, 0xf1, 0x22, 0xc0, 0x0d, 0x68, 0x73, 0x16, 0xcd, 0x5c, 0x81, 0x22, 0x50,
0xdb, 0xc6, 0x1e, 0xd4, 0x93, 0x98, 0x29, 0x36, 0xdd, 0xc3, 0xee, 0x81, 0x8c, 0xd6, 0xc1, 0x29,
0x6e, 0x19, 0x3d, 0xa8, 0x5b, 0xd1, 0x24, 0x1e, 0xd6, 0xf7, 0x6b, 0x8a, 0x0b, 0xf3, 0x2f, 0x86,
0x8d, 0x74, 0x61, 0x5f, 0x3a, 0xc3, 0xa6, 0x04, 0xd5, 0x1e, 0xb7, 0x4a, 0x1e, 0xb7, 0x4b, 0x1e,
0x77, 0xe4, 0x7a, 0x1d, 0x5a, 0x36, 0xd2, 0x0d, 0x3c, 0x36, 0x04, 0x0a, 0x41, 0x5d, 0xda, 0x43,
0xd0, 0x84, 0x98, 0xf6, 0xc5, 0x62, 0x42, 0xae, 0xf7, 0x8d, 0x5d, 0x18, 0x58, 0x8e, 0xe3, 0x72,
0x37, 0x40, 0xe2, 0xcf, 0x5c, 0x27, 0x46, 0xba, 0x35, 0x0c, 0xc1, 0x36, 0x18, 0x79, 0x7f, 0x29,
0x0c, 0x2f, 0x74, 0x72, 0x74, 0xda, 0x16, 0xc5, 0xe2, 0xdd, 0x42, 0x5e, 0xab, 0xd2, 0xff, 0x4d,
0xf2, 0x3f, 0xd3, 0x34, 0x47, 0x30, 0x9c, 0x47, 0x23, 0x4b, 0x0f, 0xe0, 0xc6, 0x13, 0xe6, 0xb1,
0x37, 0x59, 0xc2, 0x40, 0xfa, 0xd6, 0x8c, 0xa9, 0x3c, 0x0a, 0xc0, 0x79, 0x25, 0x02, 0x7c, 0x07,
0x76, 0x5e, 0xb8, 0x31, 0x5f, 0x09, 0x67, 0x7e, 0x0f, 0x90, 0x09, 0x68, 0x70, 0x6d, 0x8a, 0x5d,
0xb9, 0x9c, 0x92, 0x8b, 0x41, 0xe4, 0x76, 0x28, 0x73, 0xdb, 0x36, 0xb6, 0xa0, 0x9b, 0xf8, 0xee,
0xd5, 0xeb, 0xc0, 0x9e, 0x32, 0x2e, 0xb2, 0x2a, 0x36, 0x45, 0xee, 0xce, 0x99, 0xe7, 0x61, 0x5e,
0x71, 0x69, 0x7e, 0x01, 0xbb, 0x65, 0xfb, 0x54, 0xd6, 0xef, 0x41, 0x37, 0x8b, 0x56, 0x8c, 0xd6,
0x6a, 0x8b, 0xc3, 0x35, 0x80, 0xde, 0x6b, 0x8e, 0xd1, 0x22, 0xe2, 0xe6, 0x3e, 0x0c, 0x74, 0x8f,
0xc8, 0x03, 0x55, 0x0f, 0x16, 0x4f, 0x62, 0x72, 0x67, 0x0a, 0x2d, 0xca, 0x60, 0x5a, 0xef, 0xff,
0x5d, 0xad, 0x9a, 0x1e, 0x74, 0x34, 0x9d, 0xe5, 0x39, 0x2a, 0xcd, 0x01, 0xd9, 0xa9, 0xc6, 0xdb,
0xd0, 0x09, 0x15, 0x4f, 0xa6, 0xec, 0x74, 0x0f, 0x07, 0x44, 0x21, 0xe5, 0x9f, 0xb9, 0xd6, 0x90,
0xd6, 0xee, 0x40, 0xeb, 0xa5, 0x65, 0x9f, 0xa3, 0x31, 0x81, 0x6f, 0x87, 0xe4, 0x73, 0x5f, 0x08,
0xce, 0xd8, 0x2c, 0x88, 0xae, 0xa5, 0xbd, 0xba, 0xf9, 0x1d, 0x4e, 0x01, 0x15, 0x35, 0x0a, 0xf7,
0x6d, 0x2c, 0xce, 0x94, 0x67, 0x1a, 0xed, 0x8d, 0x34, 0xda, 0xda, 0x81, 0x5b, 0xd0, 0x9a, 0x29,
0x7c, 0xaa, 0xdf, 0x94, 0x10, 0x59, 0x35, 0x9f, 0xc0, 0xee, 0x69, 0xe8, 0xbc, 0x69, 0xea, 0x65,
0x93, 0xa5, 0x9a, 0xb2, 0x23, 0x37, 0xd4, 0x8c, 0xda, 0x83, 0x1b, 0x73, 0x28, 0x54, 0xb0, 0xeb,
0xd0, 0x7f, 0x7a, 0xc1, 0xb0, 0x22, 0xd2, 0x7c, 0xff, 0x59, 0x81, 0x86, 0xdc, 0x11, 0x1e, 0x0b,
0x32, 0x64, 0x43, 0xd9, 0xcb, 0xcd, 0x40, 0x8d, 0xdf, 0x2f, 0x45, 0xbb, 0x9e, 0x1f, 0x7d, 0x8d,
0x12, 0xc1, 0x96, 0x54, 0x40, 0xbf, 0x29, 0x15, 0x72, 0xc6, 0xcc, 0x27, 0xa2, 0x18, 0xbe, 0xce,
0x92, 0xf0, 0x15, 0x27, 0x00, 0x2c, 0x9b, 0x00, 0xbf, 0x55, 0xa0, 0xf7, 0x0d, 0xe3, 0x97, 0x41,
0x34, 0x15, 0x49, 0x8a, 0x4b, 0x2d, 0x87, 0x75, 0x1a, 0x5d, 0x8d, 0xcf, 0xae, 0x39, 0x96, 0x85,
0xcc, 0xa6, 0xf0, 0x07, 0x77, 0x8e, 0x2d, 0xd5, 0x68, 0x35, 0xb9, 0xb7, 0x09, 0x9d, 0x57, 0x57,
0x63, 0x9c, 0x82, 0x41, 0xa4, 0x7a, 0x4f, 0x8a, 0xe1, 0x96, 0x13, 0x05, 0x61, 0xc8, 0x94, 0xa7,
0x75, 0x01, 0x76, 0x92, 0x82, 0x35, 0x53, 0x29, 0xdc, 0x09, 0x09, 0xac, 0x95, 0x82, 0x9d, 0x68,
0xb0, 0x76, 0x4e, 0x2c, 0x05, 0xeb, 0xc8, 0xaa, 0x9a, 0x41, 0xfb, 0x28, 0x4c, 0x4e, 0x63, 0x6b,
0xc2, 0x44, 0xf7, 0xf3, 0x80, 0x5b, 0xde, 0x38, 0x11, 0x4b, 0x49, 0xbd, 0x6e, 0x6c, 0x43, 0x2f,
0x64, 0x11, 0xd6, 0x25, 0xed, 0x56, 0x31, 0x50, 0x75, 0xe3, 0x7f, 0xb0, 0x25, 0x97, 0x63, 0xd7,
0x1f, 0x4f, 0x59, 0xe4, 0x33, 0x6f, 0x16, 0x38, 0x8c, 0xfc, 0xd8, 0x83, 0x4d, 0x7d, 0x28, 0x9a,
0x51, 0x1e, 0x49, 0x7f, 0xcc, 0x13, 0x18, 0x9c, 0x9c, 0xe3, 0x37, 0x97, 0x7b, 0xae, 0x3f, 0x79,
0x62, 0x71, 0x4b, 0x8c, 0x7a, 0xc4, 0x77, 0x03, 0x27, 0x26, 0x83, 0xa8, 0xcd, 0x95, 0x08, 0x73,
0xc6, 0xe9, 0x91, 0x0a, 0x1a, 0xce, 0xf8, 0xec, 0x88, 0xbb, 0x33, 0x32, 0x68, 0xfe, 0x20, 0x9d,
0x50, 0x81, 0x37, 0xa1, 0x93, 0x91, 0xad, 0xc8, 0x7c, 0xad, 0xa7, 0xf9, 0x4a, 0x1d, 0x3d, 0x80,
0x75, 0xae, 0x59, 0x8c, 0xb1, 0x6a, 0x2d, 0xea, 0x8d, 0x1d, 0x92, 0x2c, 0x72, 0x34, 0x3f, 0x07,
0x78, 0x29, 0x5b, 0x51, 0x32, 0xc6, 0x79, 0x98, 0x0f, 0x10, 0x06, 0x7a, 0x66, 0x5d, 0xe9, 0xe8,
0x88, 0x2d, 0xf4, 0xe9, 0x47, 0xcb, 0xf5, 0x6c, 0xba, 0x0b, 0xd4, 0xcd, 0xbf, 0x2a, 0xd0, 0x55,
0x08, 0x8a, 0x24, 0x42, 0xd8, 0xd8, 0x7e, 0x29, 0xc4, 0x7e, 0x8a, 0x58, 0xfc, 0xc2, 0xe4, 0x6c,
0x62, 0x19, 0xc6, 0x97, 0x56, 0x48, 0x56, 0x6a, 0xcb, 0xc4, 0xee, 0x40, 0x4f, 0x65, 0x83, 0x04,
0xeb, 0xcb, 0x04, 0xef, 0x89, 0xef, 0x31, 0x32, 0x91, 0xf3, 0xaf, 0x7b, 0x78, 0xb3, 0x20, 0x21,
0x39, 0x1e, 0xc8, 0xdf, 0xa7, 0x3e, 0x8f, 0xae, 0x47, 0xf7, 0x00, 0xb2, 0x95, 0x68, 0xbb, 0x29,
0xbb, 0xa6, 0xca, 0x46, 0x4f, 0x2e, 0x2c, 0x2f, 0x21, 0xcf, 0x1f, 0x55, 0x1f, 0x56, 0xcc, 0xaf,
0x61, 0xfd, 0x4b, 0x6f, 0xea, 0x06, 0x39, 0x15, 0x94, 0x9a, 0x59, 0x3f, 0x05, 0x11, 0xf9, 0x2b,
0x96, 0xae, 0x8f, 0x4b, 0x15, 0x2e, 0xec, 0xfb, 0x20, 0xa4, 0x09, 0xaa, 0xf1, 0x54, 0xbd, 0xfc,
0x5e, 0x03, 0xc8, 0xc0, 0x8c, 0x47, 0x30, 0x72, 0x83, 0x31, 0x96, 0xd4, 0x85, 0x6b, 0x33, 0xd5,
0x02, 0xe3, 0x88, 0xd9, 0x49, 0x14, 0xbb, 0x17, 0x8c, 0x46, 0xe0, 0x2e, 0xf9, 0x52, 0xe6, 0xf0,
0x11, 0xec, 0x64, 0xba, 0x4e, 0x4e, 0xad, 0xba, 0x52, 0xed, 0x01, 0x6c, 0xa1, 0x1a, 0x0e, 0xae,
0xa4, 0xa0, 0x54, 0x5b, 0xa9, 0xf4, 0x29, 0xec, 0xe5, 0x78, 0x8a, 0x4a, 0xcd, 0xa9, 0xd6, 0x57,
0xaa, 0x7e, 0x0c, 0xbb, 0xa8, 0x7a, 0x69, 0xb9, 0xbc, 0xac, 0xd7, 0xf8, 0x07, 0x3c, 0x67, 0x2c,
0x9a, 0x14, 0x78, 0x36, 0x57, 0x2a, 0x7d, 0x00, 0x9b, 0xa8, 0x54, 0xb2, 0xd3, 0x7a, 0x93, 0x4a,
0xcc, 0x6c, 0x8e, 0x53, 0x25, 0xa7, 0xd2, 0x5e, 0xa5, 0x62, 0x3e, 0x86, 0xde, 0xf3, 0x64, 0xc2,
0xb8, 0x77, 0xa6, 0xab, 0xff, 0xdf, 0x36, 0xd0, 0x2f, 0x55, 0xe8, 0x1e, 0x4d, 0xa2, 0x20, 0x09,
0x0b, 0x5d, 0xae, 0x6a, 0x78, 0xae, 0xcb, 0x95, 0xcc, 0x5d, 0xe8, 0xa9, 0x0f, 0x28, 0x89, 0xa9,
0xe6, 0x32, 0xe6, 0x4b, 0x5d, 0x5c, 0x5c, 0xce, 0x04, 0x67, 0x12, 0x2c, 0xb6, 0x57, 0xae, 0xfc,
0x3e, 0x83, 0xfe, 0xb9, 0x72, 0x84, 0x24, 0x55, 0x2a, 0x6f, 0xa7, 0x96, 0x33, 0x82, 0x07, 0x79,
0x87, 0x55, 0x13, 0x3d, 0x87, 0xcd, 0xb9, 0xcd, 0x62, 0x2f, 0x99, 0xf9, 0x5e, 0xea, 0x1e, 0x6e,
0x11, 0x6c, 0x5e, 0x4b, 0x36, 0x58, 0x08, 0x0d, 0xc5, 0xe7, 0x7d, 0xe8, 0xfb, 0xea, 0xa3, 0xa3,
0x23, 0x51, 0xcb, 0x29, 0x16, 0x3e, 0x48, 0x18, 0x0d, 0x5b, 0xf2, 0x5b, 0x18, 0x8d, 0x7c, 0x6c,
0x31, 0x1f, 0xa2, 0x22, 0x50, 0x6c, 0x16, 0x52, 0xf8, 0x47, 0xea, 0xc6, 0xb6, 0xe8, 0xbd, 0x70,
0xf8, 0x47, 0x03, 0x6a, 0x8f, 0x8f, 0xbf, 0x32, 0x5e, 0xc1, 0x7a, 0xe9, 0xbd, 0x63, 0xa4, 0x63,
0x65, 0xf1, 0xab, 0x6a, 0xf4, 0xd6, 0xb2, 0x63, 0xba, 0x38, 0xac, 0x09, 0xcc, 0xd2, 0xad, 0x42,
0x63, 0x2e, 0xbe, 0xb3, 0x68, 0xcc, 0x65, 0x97, 0x91, 0x35, 0xe3, 0x13, 0x68, 0xaa, 0x37, 0x91,
0xb1, 0x4d, 0xb2, 0x85, 0xc7, 0xd5, 0x68, 0xa7, 0xb4, 0xab, 0x15, 0x8f, 0x00, 0xb2, 0x97, 0x84,
0x31, 0x24, 0xb1, 0xb9, 0xc7, 0xd4, 0x68, 0x6f, 0xc1, 0x89, 0x06, 0x39, 0x85, 0x8d, 0xf2, 0x53,
0xc1, 0x28, 0xc5, 0xa1, 0x7c, 0xb1, 0x1f, 0xdd, 0x5a, 0x7a, 0x9e, 0x87, 0x2d, 0x3f, 0x18, 0x34,
0xec, 0x92, 0xe7, 0x87, 0x86, 0x5d, 0xfa, 0xd2, 0x58, 0x33, 0xbe, 0x85, 0x41, 0xf1, 0xae, 0x6f,
0xfc, 0x9f, 0x94, 0x16, 0x3e, 0x41, 0x46, 0x37, 0x97, 0x9c, 0x6a, 0xc0, 0x0f, 0x55, 0xe9, 0xe2,
0x5d, 0x23, 0x8d, 0x72, 0xee, 0x21, 0x30, 0xda, 0x2e, 0x6e, 0x6a, 0xad, 0xfb, 0xd0, 0x54, 0x37,
0x48, 0x9d, 0xb2, 0xc2, 0x85, 0x72, 0xd4, 0xcb, 0xef, 0x9a, 0x6b, 0xf7, 0x2b, 0x38, 0xa5, 0xda,
0xcf, 0x18, 0x57, 0xf5, 0x9c, 0x37, 0x35, 0xa7, 0x22, 0x37, 0x85, 0xca, 0x59, 0x53, 0xfe, 0x17,
0xf0, 0xe0, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x15, 0x52, 0xba, 0x18, 0x10, 0x00, 0x00,
}

19
api/grpc/types/api.proto
View File

@ -18,20 +18,19 @@ service API {
message CreateContainerRequest {
string id = 1; // ID of container
string bundlePath = 2; // path to OCI bundle
string stdin = 3; // path to the file where stdin will be read (optional)
string stdout = 4; // path to file where stdout will be written (optional)
string stderr = 5; // path to file where stderr will be written (optional)
string console = 6; // path to the console for a container (optional)
string checkpoint = 7; // checkpoint name if you want to create immediate checkpoint (optional)
}
message CreateContainerResponse {
uint32 pid = 1; // PID of the containers main process
string stdin = 1; // path to the file where stdin will be read (optional)
string stdout = 2; // path to file where stdout will be written (optional)
string stderr = 3; // path to file where stderr will be written (optional)
}
message SignalRequest {
string id = 1; // ID of container
uint32 pid = 2; // PID of process inside container
string pid = 2; // PID of process inside container
uint32 signal = 3; // Signal which will be sent, you can find value in "man 7 signal"
}
@ -58,7 +57,6 @@ message User {
}
message AddProcessResponse {
uint32 pid = 1; // PID of process is returned in case of success
}
message CreateCheckpointRequest {
@ -101,7 +99,7 @@ message ContainerState {
}
message Process {
uint32 pid = 1;
string pid = 1;
bool terminal = 2; // Use tty for container stdio
User user = 3; // User under which process will be run
repeated string args = 4; // Arguments for process, first is binary path itself
@ -119,9 +117,8 @@ message Container {
// Machine is information about machine on which containerd is run
message Machine {
string id = 1; // ID of machine
uint32 cpus = 2; // number of cpus
uint64 memory = 3; // amount of memory
uint32 cpus = 1; // number of cpus
uint64 memory = 2; // amount of memory
}
// StateResponse is information about containerd daemon
@ -147,7 +144,7 @@ message Event {
string id = 2;
uint32 status = 3;
string bundlePath = 4;
uint32 pid = 5;
string pid = 5;
uint32 signal = 7;
Process process = 8;
repeated Container containers = 9;

210
containerd-shim/main.go Normal file
View File

@ -0,0 +1,210 @@
package main
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"os/signal"
"path/filepath"
"strconv"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/docker/containerd/util"
"github.com/opencontainers/runc/libcontainer"
"github.com/opencontainers/specs"
)
const (
bufferSize = 2048
)
type stdio struct {
stdin *os.File
stdout *os.File
stderr *os.File
console string
}
func (s *stdio) Close() error {
err := s.stdin.Close()
if oerr := s.stdout.Close(); err == nil {
err = oerr
}
if oerr := s.stderr.Close(); err == nil {
err = oerr
}
return err
}
// containerd-shim is a small shim that sits in front of a runc implementation
// that allows it to be repartented to init and handle reattach from the caller.
//
// the cwd of the shim should be the bundle for the container. Arg1 should be the path
// to the state directory where the shim can locate fifos and other information.
//
// └── shim
// ├── control
// ├── stderr
// ├── stdin
// ├── stdout
// ├── pid
// └── exit
func main() {
if len(os.Args) < 2 {
logrus.Fatal("shim: no arguments provided")
}
// start handling signals as soon as possible so that things are properly reaped
// or if runc exits before we hit the handler
signals := make(chan os.Signal, bufferSize)
signal.Notify(signals)
// set the shim as the subreaper for all orphaned processes created by the container
if err := util.SetSubreaper(1); err != nil {
logrus.WithField("error", err).Fatal("shim: set as subreaper")
}
// open the exit pipe
f, err := os.OpenFile(filepath.Join(os.Args[1], "exit"), syscall.O_WRONLY, 0)
if err != nil {
logrus.WithField("error", err).Fatal("shim: open exit pipe")
}
defer f.Close()
// open the fifos for use with the command
std, err := openContainerSTDIO(os.Args[1])
if err != nil {
logrus.WithField("error", err).Fatal("shim: open container STDIO from fifo")
}
// star the container process by invoking runc
runcPid, err := startRunc(std, os.Args[2])
if err != nil {
logrus.WithField("error", err).Fatal("shim: start runc")
}
var exitShim bool
for s := range signals {
logrus.WithField("signal", s).Debug("shim: received signal")
switch s {
case syscall.SIGCHLD:
exits, err := util.Reap()
if err != nil {
logrus.WithField("error", err).Error("shim: reaping child processes")
}
for _, e := range exits {
// check to see if runc is one of the processes that has exited
if e.Pid == runcPid {
exitShim = true
logrus.WithFields(logrus.Fields{"pid": e.Pid, "status": e.Status}).Info("shim: runc exited")
if err := writeInt(filepath.Join(os.Args[1], "exitStatus"), e.Status); err != nil {
logrus.WithFields(logrus.Fields{"error": err, "status": e.Status}).Error("shim: write exit status")
}
}
}
}
// runc has exited so the shim can also exit
if exitShim {
if err := std.Close(); err != nil {
logrus.WithField("error", err).Error("shim: close stdio")
}
if err := deleteContainer(os.Args[2]); err != nil {
logrus.WithField("error", err).Error("shim: delete runc state")
}
return
}
}
}
// startRunc starts runc detached and returns the container's pid
func startRunc(s *stdio, id string) (int, error) {
pidFile := filepath.Join(os.Args[1], "pid")
cmd := exec.Command("runc", "--id", id, "start", "-d", "--console", s.console, "--pid-file", pidFile)
cmd.Stdin = s.stdin
cmd.Stdout = s.stdout
cmd.Stderr = s.stderr
// set the parent death signal to SIGKILL so that if the shim dies the container
// process also dies
cmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGKILL,
}
if err := cmd.Run(); err != nil {
return -1, err
}
data, err := ioutil.ReadFile(pidFile)
if err != nil {
return -1, err
}
return strconv.Atoi(string(data))
}
func deleteContainer(id string) error {
return exec.Command("runc", "--id", id, "delete").Run()
}
// openContainerSTDIO opens the pre-created fifo's for use with the container
// in RDWR so that they remain open if the other side stops listening
func openContainerSTDIO(dir string) (*stdio, error) {
s := &stdio{}
spec, err := getSpec()
if err != nil {
return nil, err
}
if spec.Process.Terminal {
console, err := libcontainer.NewConsole(int(spec.Process.User.UID), int(spec.Process.User.GID))
if err != nil {
return nil, err
}
s.console = console.Path()
stdin, err := os.OpenFile(filepath.Join(dir, "stdin"), syscall.O_RDWR, 0)
if err != nil {
return nil, err
}
go func() {
io.Copy(console, stdin)
}()
stdout, err := os.OpenFile(filepath.Join(dir, "stdout"), syscall.O_RDWR, 0)
if err != nil {
return nil, err
}
go func() {
io.Copy(stdout, console)
console.Close()
}()
return s, nil
}
for name, dest := range map[string]**os.File{
"stdin": &s.stdin,
"stdout": &s.stdout,
"stderr": &s.stderr,
} {
f, err := os.OpenFile(filepath.Join(dir, name), syscall.O_RDWR, 0)
if err != nil {
return nil, err
}
*dest = f
}
return s, nil
}
func getSpec() (*specs.Spec, error) {
var s specs.Spec
f, err := os.Open("config.json")
if err != nil {
return nil, err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&s); err != nil {
return nil, err
}
return &s, nil
}
func writeInt(path string, i int) error {
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
_, err = fmt.Fprintf(f, "%d", i)
return err
}

38
containerd/main.go
View File

@ -4,6 +4,7 @@ import (
"log"
"net"
"os"
"os/signal"
"runtime"
"sync"
"syscall"
@ -29,11 +30,6 @@ const (
)
var daemonFlags = []cli.Flag{
cli.StringFlag{
Name: "id",
Value: getDefaultID(),
Usage: "unique containerd id to identify the instance",
},
cli.BoolFlag{
Name: "debug",
Usage: "enable debug output in the logs",
@ -88,7 +84,6 @@ func main() {
}
app.Action = func(context *cli.Context) {
if err := daemon(
context.String("id"),
context.String("listen"),
context.String("state-dir"),
context.Int("concurrency"),
@ -172,9 +167,12 @@ func processMetrics() {
}()
}
func daemon(id, address, stateDir string, concurrency int, oom bool) error {
func daemon(address, stateDir string, concurrency int, oom bool) error {
// setup a standard reaper so that we don't leave any zombies if we are still alive
// this is just good practice because we are spawning new processes
go reapProcesses()
tasks := make(chan *supervisor.StartTask, concurrency*100)
sv, err := supervisor.New(id, stateDir, tasks, oom)
sv, err := supervisor.New(stateDir, tasks, oom)
if err != nil {
return err
}
@ -184,17 +182,6 @@ func daemon(id, address, stateDir string, concurrency int, oom bool) error {
w := supervisor.NewWorker(sv, wg)
go w.Start()
}
// only set containerd as the subreaper if it is not an init process
if pid := os.Getpid(); pid != 1 {
logrus.WithFields(logrus.Fields{
"pid": pid,
}).Debug("containerd is not init, set as subreaper")
if err := setSubReaper(); err != nil {
return err
}
}
// start the signal handler in the background.
go startSignalHandler(sv)
if err := sv.Start(); err != nil {
return err
}
@ -211,6 +198,19 @@ func daemon(id, address, stateDir string, concurrency int, oom bool) error {
return s.Serve(l)
}
func reapProcesses() {
s := make(chan os.Signal, 2048)
signal.Notify(s, syscall.SIGCHLD)
if err := util.SetSubreaper(1); err != nil {
logrus.WithField("error", err).Error("containerd: set subpreaper")
}
for range s {
if _, err := util.Reap(); err != nil {
logrus.WithField("error", err).Error("containerd: reap child processes")
}
}
}
// getDefaultID returns the hostname for the instance host
func getDefaultID() string {
hostname, err := os.Hostname()

67
containerd/reap_linux.go
View File

@ -1,67 +0,0 @@
// +build linux
package main
import (
"os"
"os/signal"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/docker/containerd/supervisor"
"github.com/docker/containerd/util"
"github.com/opencontainers/runc/libcontainer/utils"
)
const signalBufferSize = 2048
func startSignalHandler(supervisor *supervisor.Supervisor) {
logrus.WithFields(logrus.Fields{
"bufferSize": signalBufferSize,
}).Debug("containerd: starting signal handler")
signals := make(chan os.Signal, signalBufferSize)
signal.Notify(signals)
for s := range signals {
switch s {
case syscall.SIGTERM, syscall.SIGINT:
supervisor.Stop(signals)
case syscall.SIGCHLD:
exits, err := reap()
if err != nil {
logrus.WithField("error", err).Error("containerd: reaping child processes")
}
for _, e := range exits {
supervisor.SendEvent(e)
}
}
}
supervisor.Close()
os.Exit(0)
}
func reap() (exits []*supervisor.Event, err error) {
var (
ws syscall.WaitStatus
rus syscall.Rusage
)
for {
pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, &rus)
if err != nil {
if err == syscall.ECHILD {
return exits, nil
}
return exits, err
}
if pid <= 0 {
return exits, nil
}
e := supervisor.NewEvent(supervisor.ExitEventType)
e.Pid = pid
e.Status = utils.ExitStatus(ws)
exits = append(exits, e)
}
}
func setSubReaper() error {
return util.SetSubreaper(1)
}

273
ctr/container.go
View File

@ -4,10 +4,10 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path/filepath"
"strings"
"syscall"
"text/tabwriter"
"time"
@ -15,7 +15,6 @@ import (
"github.com/codegangsta/cli"
"github.com/docker/containerd/api/grpc/types"
"github.com/docker/docker/pkg/term"
"github.com/opencontainers/runc/libcontainer"
"github.com/opencontainers/specs"
netcontext "golang.org/x/net/context"
"google.golang.org/grpc"
@ -45,6 +44,7 @@ var containersCommand = cli.Command{
listCommand,
startCommand,
statsCommand,
attachCommand,
},
Action: listContainers,
}
@ -62,15 +62,97 @@ func listContainers(context *cli.Context) {
fatal(err.Error(), 1)
}
w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0)
fmt.Fprint(w, "ID\tPATH\tSTATUS\tPID1\n")
fmt.Fprint(w, "ID\tPATH\tSTATUS\tPROCESSES\n")
for _, c := range resp.Containers {
fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", c.Id, c.BundlePath, c.Status, c.Processes[0].Pid)
procs := []string{}
for _, p := range c.Processes {
procs = append(procs, p.Pid)
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", c.Id, c.BundlePath, c.Status, strings.Join(procs, ","))
}
if err := w.Flush(); err != nil {
fatal(err.Error(), 1)
}
}
var attachCommand = cli.Command{
Name: "attach",
Usage: "attach to a running container",
Flags: []cli.Flag{
cli.StringFlag{
Name: "state-dir",
Value: "/run/containerd",
Usage: "runtime state directory",
},
cli.StringFlag{
Name: "pid,p",
Value: "init",
Usage: "specify the process id to attach to",
},
},
Action: func(context *cli.Context) {
var (
id = context.Args().First()
pid = context.String("pid")
)
if id == "" {
fatal("container id cannot be empty", 1)
}
c := getClient(context)
events, err := c.Events(netcontext.Background(), &types.EventsRequest{})
if err != nil {
fatal(err.Error(), 1)
}
type bundleState struct {
Bundle string `json:"bundle"`
}
f, err := os.Open(filepath.Join(context.String("state-dir"), id, "state.json"))
if err != nil {
fatal(err.Error(), 1)
}
var s bundleState
err = json.NewDecoder(f).Decode(&s)
f.Close()
if err != nil {
fatal(err.Error(), 1)
}
mkterm, err := readTermSetting(s.Bundle)
if err != nil {
fatal(err.Error(), 1)
}
if mkterm {
s, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
fatal(err.Error(), 1)
}
state = s
}
if err := attachStdio(
filepath.Join(context.String("state-dir"), id, pid, "stdin"),
filepath.Join(context.String("state-dir"), id, pid, "stdout"),
filepath.Join(context.String("state-dir"), id, pid, "stderr"),
); err != nil {
fatal(err.Error(), 1)
}
go func() {
io.Copy(stdin, os.Stdin)
if state != nil {
term.RestoreTerminal(os.Stdin.Fd(), state)
}
stdin.Close()
}()
for {
e, err := events.Recv()
if err != nil {
fatal(err.Error(), 1)
}
if e.Id == id && e.Type == "exit" && e.Pid == pid {
os.Exit(int(e.Status))
}
}
},
}
var startCommand = cli.Command{
Name: "start",
Usage: "start a container",
@ -110,24 +192,25 @@ var startCommand = cli.Command{
BundlePath: bpath,
Checkpoint: context.String("checkpoint"),
}
resp, err := c.CreateContainer(netcontext.Background(), r)
if err != nil {
fatal(err.Error(), 1)
}
if context.Bool("attach") {
mkterm, err := readTermSetting(bpath)
if err != nil {
fatal(err.Error(), 1)
}
if mkterm {
if err := attachTty(&r.Console); err != nil {
fatal(err.Error(), 1)
}
} else {
if err := attachStdio(&r.Stdin, &r.Stdout, &r.Stderr); err != nil {
s, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
fatal(err.Error(), 1)
}
state = s
}
if err := attachStdio(resp.Stdin, resp.Stdout, resp.Stderr); err != nil {
fatal(err.Error(), 1)
}
}
resp, err := c.CreateContainer(netcontext.Background(), r)
if err != nil {
fatal(err.Error(), 1)
}
if context.Bool("attach") {
restoreAndCloseStdin := func() {
@ -146,13 +229,11 @@ var startCommand = cli.Command{
restoreAndCloseStdin()
fatal(err.Error(), 1)
}
if e.Id == id && e.Type == "exit" {
if e.Id == id && e.Type == "exit" && e.Pid == "init" {
restoreAndCloseStdin()
os.Exit(int(e.Status))
}
}
} else {
fmt.Println(resp.Pid)
}
},
}
@ -177,69 +258,24 @@ func readTermSetting(path string) (bool, error) {
return spec.Process.Terminal, nil
}
func attachTty(consolePath *string) error {
console, err := libcontainer.NewConsole(os.Getuid(), os.Getgid())
func attachStdio(stdins, stdout, stderr string) error {
stdinf, err := os.OpenFile(stdins, syscall.O_RDWR, 0)
if err != nil {
return err
}
*consolePath = console.Path()
stdin = console
go func() {
io.Copy(os.Stdout, console)
console.Close()
}()
s, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
return err
}
state = s
return nil
}
stdin = stdinf
func attachStdio(stdins, stdout, stderr *string) error {
dir, err := ioutil.TempDir("", "ctr-")
stdoutf, err := os.OpenFile(stdout, syscall.O_RDWR, 0)
if err != nil {
return err
}
for _, p := range []struct {
path string
flag int
done func(f *os.File)
}{
{
path: filepath.Join(dir, "stdin"),
flag: syscall.O_RDWR,
done: func(f *os.File) {
*stdins = filepath.Join(dir, "stdin")
stdin = f
},
},
{
path: filepath.Join(dir, "stdout"),
flag: syscall.O_RDWR,
done: func(f *os.File) {
*stdout = filepath.Join(dir, "stdout")
go io.Copy(os.Stdout, f)
},
},
{
path: filepath.Join(dir, "stderr"),
flag: syscall.O_RDWR,
done: func(f *os.File) {
*stderr = filepath.Join(dir, "stderr")
go io.Copy(os.Stderr, f)
},
},
} {
if err := syscall.Mkfifo(p.path, 0755); err != nil {
return fmt.Errorf("mkfifo: %s %v", p.path, err)
}
f, err := os.OpenFile(p.path, p.flag, 0)
if err != nil {
return fmt.Errorf("open: %s %v", p.path, err)
}
p.done(f)
go io.Copy(os.Stdout, stdoutf)
stderrf, err := os.OpenFile(stderr, syscall.O_RDWR, 0)
if err != nil {
return err
}
go io.Copy(os.Stderr, stderrf)
return nil
}
@ -247,7 +283,7 @@ var killCommand = cli.Command{
Name: "kill",
Usage: "send a signal to a container or its processes",
Flags: []cli.Flag{
cli.IntFlag{
cli.StringFlag{
Name: "pid,p",
Usage: "pid of the process to signal within the container",
},
@ -265,7 +301,7 @@ var killCommand = cli.Command{
c := getClient(context)
if _, err := c.Signal(netcontext.Background(), &types.SignalRequest{
Id: id,
Pid: uint32(context.Int("pid")),
Pid: context.String("pid"),
Signal: uint32(context.Int("signal")),
}); err != nil {
fatal(err.Error(), 1)
@ -308,55 +344,58 @@ var execCommand = cli.Command{
},
},
Action: func(context *cli.Context) {
p := &types.AddProcessRequest{
Args: context.Args(),
Cwd: context.String("cwd"),
Terminal: context.Bool("tty"),
Id: context.String("id"),
Env: context.StringSlice("env"),
User: &types.User{
Uid: uint32(context.Int("uid")),
Gid: uint32(context.Int("gid")),
},
}
c := getClient(context)
events, err := c.Events(netcontext.Background(), &types.EventsRequest{})
if err != nil {
fatal(err.Error(), 1)
}
if context.Bool("attach") {
if p.Terminal {
if err := attachTty(&p.Console); err != nil {
fatal(err.Error(), 1)
}
} else {
if err := attachStdio(&p.Stdin, &p.Stdout, &p.Stderr); err != nil {
fatal(err.Error(), 1)
}
panic("not implemented")
/*
p := &types.AddProcessRequest{
Args: context.Args(),
Cwd: context.String("cwd"),
Terminal: context.Bool("tty"),
Id: context.String("id"),
Env: context.StringSlice("env"),
User: &types.User{
Uid: uint32(context.Int("uid")),
Gid: uint32(context.Int("gid")),
},
}
}
r, err := c.AddProcess(netcontext.Background(), p)
if err != nil {
fatal(err.Error(), 1)
}
if context.Bool("attach") {
go func() {
io.Copy(stdin, os.Stdin)
if state != nil {
term.RestoreTerminal(os.Stdin.Fd(), state)
}
stdin.Close()
}()
for {
e, err := events.Recv()
c := getClient(context)
events, err := c.Events(netcontext.Background(), &types.EventsRequest{})
if err != nil {
fatal(err.Error(), 1)
}
if context.Bool("attach") {
if p.Terminal {
if err := attachTty(&p.Console); err != nil {
fatal(err.Error(), 1)
}
} else {
if err := attachStdio(&p.Stdin, &p.Stdout, &p.Stderr); err != nil {
fatal(err.Error(), 1)
}
}
}
r, err := c.AddProcess(netcontext.Background(), p)
if err != nil {
fatal(err.Error(), 1)
}
if e.Pid == r.Pid && e.Type == "exit" {
os.Exit(int(e.Status))
if context.Bool("attach") {
go func() {
io.Copy(stdin, os.Stdin)
if state != nil {
term.RestoreTerminal(os.Stdin.Fd(), state)
}
stdin.Close()
}()
for {
e, err := events.Recv()
if err != nil {
fatal(err.Error(), 1)
}
if e.Pid == r.Pid && e.Type == "exit" {
os.Exit(int(e.Status))
}
}
}
}
}
*/
},
}

2
ctr/events.go
View File

@ -27,7 +27,7 @@ var eventsCommand = cli.Command{
if err != nil {
fatal(err.Error(), 1)
}
fmt.Fprintf(w, "%s\t%s\t%d\t%d\n", e.Type, e.Id, e.Pid, e.Status)
fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", e.Type, e.Id, e.Pid, e.Status)
w.Flush()
}
},

955
linux/linux.go
View File

@ -1,955 +0,0 @@
// +build libcontainer
package linux
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
goruntime "runtime"
"strconv"
"strings"
"syscall"
"time"
"github.com/docker/containerd/runtime"
"github.com/opencontainers/runc/libcontainer"
"github.com/opencontainers/runc/libcontainer/configs"
_ "github.com/opencontainers/runc/libcontainer/nsenter"
"github.com/opencontainers/runc/libcontainer/seccomp"
"github.com/opencontainers/specs"
)
const (
RLIMIT_CPU = iota // CPU time in sec
RLIMIT_FSIZE // Maximum filesize
RLIMIT_DATA // max data size
RLIMIT_STACK // max stack size
RLIMIT_CORE // max core file size
RLIMIT_RSS // max resident set size
RLIMIT_NPROC // max number of processes
RLIMIT_NOFILE // max number of open files
RLIMIT_MEMLOCK // max locked-in-memory address space
RLIMIT_AS // address space limit
RLIMIT_LOCKS // maximum file locks held
RLIMIT_SIGPENDING // max number of pending signals
RLIMIT_MSGQUEUE // maximum bytes in POSIX mqueues
RLIMIT_NICE // max nice prio allowed to raise to
RLIMIT_RTPRIO // maximum realtime priority
RLIMIT_RTTIME // timeout for RT tasks in us
)
var rlimitMap = map[string]int{
"RLIMIT_CPU": RLIMIT_CPU,
"RLIMIT_FSIZE": RLIMIT_FSIZE,
"RLIMIT_DATA": RLIMIT_DATA,
"RLIMIT_STACK": RLIMIT_STACK,
"RLIMIT_CORE": RLIMIT_CORE,
"RLIMIT_RSS": RLIMIT_RSS,
"RLIMIT_NPROC": RLIMIT_NPROC,
"RLIMIT_NOFILE": RLIMIT_NOFILE,
"RLIMIT_MEMLOCK": RLIMIT_MEMLOCK,
"RLIMIT_AS": RLIMIT_AS,
"RLIMIT_LOCKS": RLIMIT_LOCKS,
"RLIMIT_SGPENDING": RLIMIT_SIGPENDING,
"RLIMIT_MSGQUEUE": RLIMIT_MSGQUEUE,
"RLIMIT_NICE": RLIMIT_NICE,
"RLIMIT_RTPRIO": RLIMIT_RTPRIO,
"RLIMIT_RTTIME": RLIMIT_RTTIME,
}
func strToRlimit(key string) (int, error) {
rl, ok := rlimitMap[key]
if !ok {
return 0, fmt.Errorf("Wrong rlimit value: %s", key)
}
return rl, nil
}
const wildcard = -1
var allowedDevices = []*configs.Device{
// allow mknod for any device
{
Type: 'c',
Major: wildcard,
Minor: wildcard,
Permissions: "m",
},
{
Type: 'b',
Major: wildcard,
Minor: wildcard,
Permissions: "m",
},
{
Path: "/dev/console",
Type: 'c',
Major: 5,
Minor: 1,
Permissions: "rwm",
},
{
Path: "/dev/tty0",
Type: 'c',
Major: 4,
Minor: 0,
Permissions: "rwm",
},
{
Path: "/dev/tty1",
Type: 'c',
Major: 4,
Minor: 1,
Permissions: "rwm",
},
// /dev/pts/ - pts namespaces are "coming soon"
{
Path: "",
Type: 'c',
Major: 136,
Minor: wildcard,
Permissions: "rwm",
},
{
Path: "",
Type: 'c',
Major: 5,
Minor: 2,
Permissions: "rwm",
},
// tuntap
{
Path: "",
Type: 'c',
Major: 10,
Minor: 200,
Permissions: "rwm",
},
}
var namespaceMapping = map[specs.NamespaceType]configs.NamespaceType{
specs.PIDNamespace: configs.NEWPID,
specs.NetworkNamespace: configs.NEWNET,
specs.MountNamespace: configs.NEWNS,
specs.UserNamespace: configs.NEWUSER,
specs.IPCNamespace: configs.NEWIPC,
specs.UTSNamespace: configs.NEWUTS,
}
var mountPropagationMapping = map[string]int{
"rprivate": syscall.MS_PRIVATE | syscall.MS_REC,
"private": syscall.MS_PRIVATE,
"rslave": syscall.MS_SLAVE | syscall.MS_REC,
"slave": syscall.MS_SLAVE,
"rshared": syscall.MS_SHARED | syscall.MS_REC,
"shared": syscall.MS_SHARED,
"": syscall.MS_PRIVATE | syscall.MS_REC,
}
func init() {
if len(os.Args) > 1 && os.Args[1] == "init" {
goruntime.GOMAXPROCS(1)
goruntime.LockOSThread()
factory, _ := libcontainer.New("")
if err := factory.StartInitialization(); err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
panic("--this line should have never been executed, congratulations--")
}
}
type libcontainerProcess struct {
process *libcontainer.Process
spec specs.Process
}
// change interface to support an error
func (p *libcontainerProcess) Pid() (int, error) {
pid, err := p.process.Pid()
if err != nil {
return -1, err
}
return pid, nil
}
func (p *libcontainerProcess) Spec() specs.Process {
return p.spec
}
func (p *libcontainerProcess) Signal(s os.Signal) error {
return p.process.Signal(s)
}
func (p *libcontainerProcess) Close() error {
// in close we always need to call wait to close/flush any pipes
p.process.Wait()
// explicitly close any open fd on the process
for _, cl := range []interface{}{
p.process.Stderr,
p.process.Stdout,
p.process.Stdin,
} {
if cl != nil {
if c, ok := cl.(io.Closer); ok {
c.Close()
}
}
}
return nil
}
type libcontainerContainer struct {
c libcontainer.Container
initProcess *libcontainerProcess
additionalProcesses map[int]*libcontainerProcess
exitStatus int
exited bool
path string
}
func (c *libcontainerContainer) Checkpoints() ([]runtime.Checkpoint, error) {
out := []runtime.Checkpoint{}
files, err := ioutil.ReadDir(c.getCheckpointPath(""))
if err != nil {
if os.IsNotExist(err) {
return out, nil
}
return nil, err
}
for _, fi := range files {
out = append(out, runtime.Checkpoint{
Name: fi.Name(),
Timestamp: fi.ModTime(),
})
}
return out, nil
}
func (c *libcontainerContainer) DeleteCheckpoint(name string) error {
path := c.getCheckpointPath(name)
if err := os.RemoveAll(path); err != nil {
if os.IsNotExist(err) {
return runtime.ErrCheckpointNotExists
}
return err
}
return nil
}
func (c *libcontainerContainer) getCheckpointPath(name string) string {
return filepath.Join(c.path, "checkpoints", name)
}
func (c *libcontainerContainer) Checkpoint(cp runtime.Checkpoint) error {
opts := c.createCheckpointOpts(cp)
if err := os.MkdirAll(filepath.Dir(opts.ImagesDirectory), 0755); err != nil {
return err
}
// mkdir is atomic so if it already exists we can fail
if err := os.Mkdir(opts.ImagesDirectory, 0755); err != nil {
if os.IsExist(err) {
return runtime.ErrCheckpointExists
}
return err
}
if err := c.c.Checkpoint(opts); err != nil {
return err
}
return nil
}
func (c *libcontainerContainer) createCheckpointOpts(cp runtime.Checkpoint) *libcontainer.CriuOpts {
opts := libcontainer.CriuOpts{}
opts.LeaveRunning = !cp.Exit
opts.ShellJob = cp.Shell
opts.TcpEstablished = cp.Tcp
opts.ExternalUnixConnections = cp.UnixSockets
opts.ImagesDirectory = c.getCheckpointPath(cp.Name)
return &opts
}
func (c *libcontainerContainer) Restore(name string) error {
path := c.getCheckpointPath(name)
var opts libcontainer.CriuOpts
opts.ImagesDirectory = path
return c.c.Restore(c.initProcess.process, &opts)
}
func (c *libcontainerContainer) Resume() error {
return c.c.Resume()
}
func (c *libcontainerContainer) Pause() error {
return c.c.Pause()
}
func (c *libcontainerContainer) State() runtime.State {
// TODO: what to do with error
state, err := c.c.Status()
if err != nil {
return runtime.State("")
}
switch state {
case libcontainer.Paused, libcontainer.Pausing:
return runtime.Paused
}
return runtime.State("")
}
func (c *libcontainerContainer) ID() string {
return c.c.ID()
}
func (c *libcontainerContainer) Path() string {
return c.path
}
func (c *libcontainerContainer) Pid() (int, error) {
return c.initProcess.Pid()
}
func (c *libcontainerContainer) Start() error {
return c.c.Start(c.initProcess.process)
}
func (c *libcontainerContainer) SetExited(status int) {
c.exitStatus = status
// meh
c.exited = true
c.initProcess.Close()
}
func (c *libcontainerContainer) Stats() (*runtime.Stat, error) {
now := time.Now()
stats, err := c.c.Stats()
if err != nil {
return nil, err
}
return &runtime.Stat{
Timestamp: now,
Data: stats,
}, nil
}
func (c *libcontainerContainer) Delete() error {
return c.c.Destroy()
}
func (c *libcontainerContainer) Processes() ([]runtime.Process, error) {
procs := []runtime.Process{
c.initProcess,
}
for _, p := range c.additionalProcesses {
procs = append(procs, p)
}
return procs, nil
}
func (c *libcontainerContainer) RemoveProcess(pid int) error {
proc, ok := c.additionalProcesses[pid]
if !ok {
return runtime.ErrNotChildProcess
}
err := proc.Close()
delete(c.additionalProcesses, pid)
return err
}
func (c *libcontainerContainer) OOM() (<-chan struct{}, error) {
return c.c.NotifyOOM()
}
func NewRuntime(stateDir string) (runtime.Runtime, error) {
f, err := libcontainer.New(stateDir, libcontainer.Cgroupfs, func(l *libcontainer.LinuxFactory) error {
//l.CriuPath = context.GlobalString("criu")
return nil
})
if err != nil {
return nil, err
}
return &libcontainerRuntime{
factory: f,
}, nil
}
type libcontainerRuntime struct {
factory libcontainer.Factory
}
func (r *libcontainerRuntime) Type() string {
return "libcontainer"
}
func (r *libcontainerRuntime) Create(id, bundlePath, consolePath string) (runtime.Container, *runtime.IO, error) {
spec, rspec, err := r.loadSpec(
filepath.Join(bundlePath, "config.json"),
filepath.Join(bundlePath, "runtime.json"),
)
if err != nil {
return nil, nil, err
}
config, err := r.createLibcontainerConfig(id, bundlePath, spec, rspec)
if err != nil {
return nil, nil, err
}
container, err := r.factory.Create(id, config)
if err != nil {
return nil, nil, fmt.Errorf("create container: %v", err)
}
process, err := r.newProcess(spec.Process)
if err != nil {
return nil, nil, err
}
var rio runtime.IO
if spec.Process.Terminal {
if err := process.ConsoleFromPath(consolePath); err != nil {
return nil, nil, err
}
} else {
uid, err := config.HostUID()
if err != nil {
return nil, nil, err
}
i, err := process.InitializeIO(uid)
if err != nil {
return nil, nil, err
}
rio.Stdin = i.Stdin
rio.Stderr = i.Stderr
rio.Stdout = i.Stdout
}
c := &libcontainerContainer{
c: container,
additionalProcesses: make(map[int]*libcontainerProcess),
initProcess: &libcontainerProcess{
process: process,
spec: spec.Process,
},
path: bundlePath,
}
return c, &rio, nil
}
func (r *libcontainerRuntime) StartProcess(ci runtime.Container, p specs.Process, consolePath string) (runtime.Process, *runtime.IO, error) {
c, ok := ci.(*libcontainerContainer)
if !ok {
return nil, nil, runtime.ErrInvalidContainerType
}
process, err := r.newProcess(p)
if err != nil {
return nil, nil, err
}
var rio runtime.IO
if p.Terminal {
if err := process.ConsoleFromPath(consolePath); err != nil {
return nil, nil, err
}
} else {
uid, err := c.c.Config().HostUID()
if err != nil {
return nil, nil, err
}
i, err := process.InitializeIO(uid)
if err != nil {
return nil, nil, err
}
rio.Stdin = i.Stdin
rio.Stderr = i.Stderr
rio.Stdout = i.Stdout
}
if err := c.c.Start(process); err != nil {
return nil, nil, err
}
lp := &libcontainerProcess{
process: process,
spec: p,
}
pid, err := process.Pid()
if err != nil {
return nil, nil, err
}
c.additionalProcesses[pid] = lp
return lp, &rio, nil
}
// newProcess returns a new libcontainer Process with the arguments from the
// spec and stdio from the current process.
func (r *libcontainerRuntime) newProcess(p specs.Process) (*libcontainer.Process, error) {
return &libcontainer.Process{
Args: p.Args,
Env: p.Env,
// TODO: fix libcontainer's API to better support uid/gid in a typesafe way.
User: fmt.Sprintf("%d:%d", p.User.UID, p.User.GID),
Cwd: p.Cwd,
}, nil
}
// loadSpec loads the specification from the provided path.
// If the path is empty then the default path will be "config.json"
func (r *libcontainerRuntime) loadSpec(cPath, rPath string) (spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec, err error) {
cf, err := os.Open(cPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil, fmt.Errorf("JSON specification file at %s not found", cPath)
}
return spec, rspec, err
}
defer cf.Close()
rf, err := os.Open(rPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil, fmt.Errorf("JSON runtime config file at %s not found", rPath)
}
return spec, rspec, err
}
defer rf.Close()
if err = json.NewDecoder(cf).Decode(&spec); err != nil {
return spec, rspec, fmt.Errorf("unmarshal %s: %v", cPath, err)
}
if err = json.NewDecoder(rf).Decode(&rspec); err != nil {
return spec, rspec, fmt.Errorf("unmarshal %s: %v", rPath, err)
}
return spec, rspec, r.checkSpecVersion(spec)
}
// checkSpecVersion makes sure that the spec version matches runc's while we are in the initial
// development period. It is better to hard fail than have missing fields or options in the spec.
func (r *libcontainerRuntime) checkSpecVersion(s *specs.LinuxSpec) error {
if s.Version != specs.Version {
return fmt.Errorf("spec version is not compatible with implemented version %q: spec %q", specs.Version, s.Version)
}
return nil
}
func (r *libcontainerRuntime) createLibcontainerConfig(cgroupName, bundlePath string, spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec) (*configs.Config, error) {
rootfsPath := spec.Root.Path
if !filepath.IsAbs(rootfsPath) {
rootfsPath = filepath.Join(bundlePath, rootfsPath)
}
config := &configs.Config{
Rootfs: rootfsPath,
Capabilities: spec.Linux.Capabilities,
Readonlyfs: spec.Root.Readonly,
Hostname: spec.Hostname,
}
for _, ns := range rspec.Linux.Namespaces {
t, exists := namespaceMapping[ns.Type]
if !exists {
return nil, fmt.Errorf("namespace %q does not exist", ns)
}
config.Namespaces.Add(t, ns.Path)
}
if config.Namespaces.Contains(configs.NEWNET) {
config.Networks = []*configs.Network{
{
Type: "loopback",
},
}
}
for _, mp := range spec.Mounts {
m, ok := rspec.Mounts[mp.Name]
if !ok {
return nil, fmt.Errorf("Mount with Name %q not found in runtime config", mp.Name)
}
config.Mounts = append(config.Mounts, r.createLibcontainerMount(bundlePath, mp.Path, m))
}
// Convert rootfs propagation flag
if rspec.Linux.RootfsPropagation != "" {
_, pflags, _ := parseMountOptions([]string{rspec.Linux.RootfsPropagation})
if len(pflags) == 1 {
config.RootPropagation = pflags[0]
}
}
if err := r.createDevices(rspec, config); err != nil {
return nil, err
}
if err := r.setupUserNamespace(rspec, config); err != nil {
return nil, err
}
for _, rlimit := range rspec.Linux.Rlimits {
rl, err := r.createLibContainerRlimit(rlimit)
if err != nil {
return nil, err
}
config.Rlimits = append(config.Rlimits, rl)
}
c, err := r.createCgroupConfig(cgroupName, rspec, config.Devices)
if err != nil {
return nil, err
}
config.Cgroups = c
if config.Readonlyfs {
r.setReadonly(config)
config.MaskPaths = []string{
"/proc/kcore",
}
config.ReadonlyPaths = []string{
"/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus",
}
}
seccomp, err := r.setupSeccomp(&rspec.Linux.Seccomp)
if err != nil {
return nil, err
}
config.Seccomp = seccomp
config.Sysctl = rspec.Linux.Sysctl
config.ProcessLabel = rspec.Linux.SelinuxProcessLabel
config.AppArmorProfile = rspec.Linux.ApparmorProfile
for _, g := range spec.Process.User.AdditionalGids {
config.AdditionalGroups = append(config.AdditionalGroups, strconv.FormatUint(uint64(g), 10))
}
r.createHooks(rspec, config)
config.Version = specs.Version
return config, nil
}
func (r *libcontainerRuntime) createLibcontainerMount(cwd, dest string, m specs.Mount) *configs.Mount {
flags, pgflags, data := parseMountOptions(m.Options)
source := m.Source
if m.Type == "bind" {
if !filepath.IsAbs(source) {
source = filepath.Join(cwd, m.Source)
}
}
return &configs.Mount{
Device: m.Type,
Source: source,
Destination: dest,
Data: data,
Flags: flags,
PropagationFlags: pgflags,
}
}
func (rt *libcontainerRuntime) createCgroupConfig(name string, spec *specs.LinuxRuntimeSpec, devices []*configs.Device) (*configs.Cgroup, error) {
cr := &configs.Cgroup{
Name: name,
Parent: "/containerd",
}
c := &configs.Resources{
AllowedDevices: append(devices, allowedDevices...),
}
cr.Resources = c
r := spec.Linux.Resources
if r.Memory != nil {
if r.Memory.Limit != nil {
c.Memory = int64(*r.Memory.Limit)
}
if r.Memory.Reservation != nil {
c.MemoryReservation = int64(*r.Memory.Reservation)
}
if r.Memory.Swap != nil {
c.MemorySwap = int64(*r.Memory.Swap)
}
if r.Memory.Kernel != nil {
c.KernelMemory = int64(*r.Memory.Kernel)
}
if r.Memory.Swappiness != nil {
c.MemorySwappiness = int64(*r.Memory.Swappiness)
}
}
if r.CPU != nil {
if r.CPU.Shares != nil {
c.CpuShares = int64(*r.CPU.Shares)
}
if r.CPU.Quota != nil {
c.CpuQuota = int64(*r.CPU.Quota)
}
if r.CPU.Period != nil {
c.CpuPeriod = int64(*r.CPU.Period)
}
if r.CPU.RealtimeRuntime != nil {
c.CpuRtRuntime = int64(*r.CPU.RealtimeRuntime)
}
if r.CPU.RealtimePeriod != nil {
c.CpuRtPeriod = int64(*r.CPU.RealtimePeriod)
}
if r.CPU.Cpus != nil {
c.CpusetCpus = *r.CPU.Cpus
}
if r.CPU.Mems != nil {
c.CpusetMems = *r.CPU.Mems
}
}
if r.BlockIO != nil {
if r.BlockIO.Weight != nil {
c.BlkioWeight = *r.BlockIO.Weight
}
if r.BlockIO.LeafWeight != nil {
c.BlkioLeafWeight = *r.BlockIO.LeafWeight
}
}
for _, wd := range r.BlockIO.WeightDevice {
weightDevice := configs.NewWeightDevice(wd.Major, wd.Minor, *wd.Weight, *wd.LeafWeight)
c.BlkioWeightDevice = append(c.BlkioWeightDevice, weightDevice)
}
for _, td := range r.BlockIO.ThrottleReadBpsDevice {
throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate)
c.BlkioThrottleReadBpsDevice = append(c.BlkioThrottleReadBpsDevice, throttleDevice)
}
for _, td := range r.BlockIO.ThrottleWriteBpsDevice {
throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate)
c.BlkioThrottleWriteBpsDevice = append(c.BlkioThrottleWriteBpsDevice, throttleDevice)
}
for _, td := range r.BlockIO.ThrottleReadIOPSDevice {
throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate)
c.BlkioThrottleReadIOPSDevice = append(c.BlkioThrottleReadIOPSDevice, throttleDevice)
}
for _, td := range r.BlockIO.ThrottleWriteIOPSDevice {
throttleDevice := configs.NewThrottleDevice(td.Major, td.Minor, *td.Rate)
c.BlkioThrottleWriteIOPSDevice = append(c.BlkioThrottleWriteIOPSDevice, throttleDevice)
}
for _, l := range r.HugepageLimits {
c.HugetlbLimit = append(c.HugetlbLimit, &configs.HugepageLimit{
Pagesize: *l.Pagesize,
Limit: *l.Limit,
})
}
c.OomKillDisable = r.DisableOOMKiller != nil && *r.DisableOOMKiller
if r.Network != nil {
c.NetClsClassid = r.Network.ClassID
for _, m := range r.Network.Priorities {
c.NetPrioIfpriomap = append(c.NetPrioIfpriomap, &configs.IfPrioMap{
Interface: m.Name,
Priority: int64(m.Priority),
})
}
}
return cr, nil
}
func (r *libcontainerRuntime) createDevices(spec *specs.LinuxRuntimeSpec, config *configs.Config) error {
for _, d := range spec.Linux.Devices {
device := &configs.Device{
Type: d.Type,
Path: d.Path,
Major: d.Major,
Minor: d.Minor,
Permissions: d.Permissions,
FileMode: d.FileMode,
Uid: d.UID,
Gid: d.GID,
}
config.Devices = append(config.Devices, device)
}
return nil
}
func (r *libcontainerRuntime) setReadonly(config *configs.Config) {
for _, m := range config.Mounts {
if m.Device == "sysfs" {
m.Flags |= syscall.MS_RDONLY
}
}
}
func (r *libcontainerRuntime) setupUserNamespace(spec *specs.LinuxRuntimeSpec, config *configs.Config) error {
if len(spec.Linux.UIDMappings) == 0 {
return nil
}
config.Namespaces.Add(configs.NEWUSER, "")
create := func(m specs.IDMapping) configs.IDMap {
return configs.IDMap{
HostID: int(m.HostID),
ContainerID: int(m.ContainerID),
Size: int(m.Size),
}
}
for _, m := range spec.Linux.UIDMappings {
config.UidMappings = append(config.UidMappings, create(m))
}
for _, m := range spec.Linux.GIDMappings {
config.GidMappings = append(config.GidMappings, create(m))
}
rootUID, err := config.HostUID()
if err != nil {
return err
}
rootGID, err := config.HostGID()
if err != nil {
return err
}
for _, node := range config.Devices {
node.Uid = uint32(rootUID)
node.Gid = uint32(rootGID)
}
return nil
}
func (r *libcontainerRuntime) createLibContainerRlimit(rlimit specs.Rlimit) (configs.Rlimit, error) {
rl, err := strToRlimit(rlimit.Type)
if err != nil {
return configs.Rlimit{}, err
}
return configs.Rlimit{
Type: rl,
Hard: uint64(rlimit.Hard),
Soft: uint64(rlimit.Soft),
}, nil
}
// parseMountOptions parses the string and returns the flags, propagation
// flags and any mount data that it contains.
func parseMountOptions(options []string) (int, []int, string) {
var (
flag int
pgflag []int
data []string
)
flags := map[string]struct {
clear bool
flag int
}{
"async": {true, syscall.MS_SYNCHRONOUS},
"atime": {true, syscall.MS_NOATIME},
"bind": {false, syscall.MS_BIND},
"defaults": {false, 0},
"dev": {true, syscall.MS_NODEV},
"diratime": {true, syscall.MS_NODIRATIME},
"dirsync": {false, syscall.MS_DIRSYNC},
"exec": {true, syscall.MS_NOEXEC},
"mand": {false, syscall.MS_MANDLOCK},
"noatime": {false, syscall.MS_NOATIME},
"nodev": {false, syscall.MS_NODEV},
"nodiratime": {false, syscall.MS_NODIRATIME},
"noexec": {false, syscall.MS_NOEXEC},
"nomand": {true, syscall.MS_MANDLOCK},
"norelatime": {true, syscall.MS_RELATIME},
"nostrictatime": {true, syscall.MS_STRICTATIME},
"nosuid": {false, syscall.MS_NOSUID},
"rbind": {false, syscall.MS_BIND | syscall.MS_REC},
"relatime": {false, syscall.MS_RELATIME},
"remount": {false, syscall.MS_REMOUNT},
"ro": {false, syscall.MS_RDONLY},
"rw": {true, syscall.MS_RDONLY},
"strictatime": {false, syscall.MS_STRICTATIME},
"suid": {true, syscall.MS_NOSUID},
"sync": {false, syscall.MS_SYNCHRONOUS},
}
propagationFlags := map[string]struct {
clear bool
flag int
}{
"private": {false, syscall.MS_PRIVATE},
"shared": {false, syscall.MS_SHARED},
"slave": {false, syscall.MS_SLAVE},
"unbindable": {false, syscall.MS_UNBINDABLE},
"rprivate": {false, syscall.MS_PRIVATE | syscall.MS_REC},
"rshared": {false, syscall.MS_SHARED | syscall.MS_REC},
"rslave": {false, syscall.MS_SLAVE | syscall.MS_REC},
"runbindable": {false, syscall.MS_UNBINDABLE | syscall.MS_REC},
}
for _, o := range options {
// If the option does not exist in the flags table or the flag
// is not supported on the platform,
// then it is a data value for a specific fs type
if f, exists := flags[o]; exists && f.flag != 0 {
if f.clear {
flag &= ^f.flag
} else {
flag |= f.flag
}
} else if f, exists := propagationFlags[o]; exists && f.flag != 0 {
pgflag = append(pgflag, f.flag)
} else {
data = append(data, o)
}
}
return flag, pgflag, strings.Join(data, ",")
}
func (r *libcontainerRuntime) setupSeccomp(config *specs.Seccomp) (*configs.Seccomp, error) {
if config == nil {
return nil, nil
}
// No default action specified, no syscalls listed, assume seccomp disabled
if config.DefaultAction == "" && len(config.Syscalls) == 0 {
return nil, nil
}
newConfig := new(configs.Seccomp)
newConfig.Syscalls = []*configs.Syscall{}
if len(config.Architectures) > 0 {
newConfig.Architectures = []string{}
for _, arch := range config.Architectures {
newArch, err := seccomp.ConvertStringToArch(string(arch))
if err != nil {
return nil, err
}
newConfig.Architectures = append(newConfig.Architectures, newArch)
}
}
// Convert default action from string representation
newDefaultAction, err := seccomp.ConvertStringToAction(string(config.DefaultAction))
if err != nil {
return nil, err
}
newConfig.DefaultAction = newDefaultAction
// Loop through all syscall blocks and convert them to libcontainer format
for _, call := range config.Syscalls {
newAction, err := seccomp.ConvertStringToAction(string(call.Action))
if err != nil {
return nil, err
}
newCall := configs.Syscall{
Name: call.Name,
Action: newAction,
Args: []*configs.Arg{},
}
// Loop through all the arguments of the syscall and convert them
for _, arg := range call.Args {
newOp, err := seccomp.ConvertStringToOperator(string(arg.Op))
if err != nil {
return nil, err
}
newArg := configs.Arg{
Index: arg.Index,
Value: arg.Value,
ValueTwo: arg.ValueTwo,
Op: newOp,
}
newCall.Args = append(newCall.Args, &newArg)
}
newConfig.Syscalls = append(newConfig.Syscalls, &newCall)
}
return newConfig, nil
}
func (r *libcontainerRuntime) createHooks(rspec *specs.LinuxRuntimeSpec, config *configs.Config) {
config.Hooks = &configs.Hooks{}
for _, h := range rspec.Hooks.Prestart {
cmd := configs.Command{
Path: h.Path,
Args: h.Args,
Env: h.Env,
}
config.Hooks.Prestart = append(config.Hooks.Prestart, configs.NewCommandHook(cmd))
}
for _, h := range rspec.Hooks.Poststop {
cmd := configs.Command{
Path: h.Path,
Args: h.Args,
Env: h.Env,
}
config.Hooks.Poststop = append(config.Hooks.Poststop, configs.NewCommandHook(cmd))
}
}

235
runc/runc.go
View File

@ -1,235 +0,0 @@
// +build runc
package runc
import (
"encoding/json"
"errors"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"github.com/docker/containerd/runtime"
"github.com/opencontainers/specs"
)
func NewRuntime(stateDir string) (runtime.Runtime, error) {
return &runcRuntime{
stateDir: stateDir,
}, nil
}
type runcContainer struct {
id string
path string
stateDir string
exitStatus int
processes map[int]*runcProcess
initProcess *runcProcess
}
func (c *runcContainer) ID() string {
return c.id
}
func (c *runcContainer) Start() error {
return c.initProcess.cmd.Start()
}
func (c *runcContainer) Stats() (*runtime.Stat, error) {
return nil, errors.New("containerd: runc does not support stats in containerd")
}
func (c *runcContainer) Path() string {
return c.path
}
func (c *runcContainer) Pid() (int, error) {
return c.initProcess.cmd.Process.Pid, nil
}
func (c *runcContainer) SetExited(status int) {
c.exitStatus = status
}
// noop for runc
func (c *runcContainer) Delete() error {
return nil
}
func (c *runcContainer) Processes() ([]runtime.Process, error) {
procs := []runtime.Process{
c.initProcess,
}
for _, p := range c.processes {
procs = append(procs, p)
}
return procs, nil
}
func (c *runcContainer) RemoveProcess(pid int) error {
if _, ok := c.processes[pid]; !ok {
return runtime.ErrNotChildProcess
}
delete(c.processes, pid)
return nil
}
func (c *runcContainer) State() runtime.State {
// TODO: how to do this with runc
return runtime.State{
Status: runtime.Running,
}
}
func (c *runcContainer) Resume() error {
return c.newCommand("resume").Run()
}
func (c *runcContainer) Pause() error {
return c.newCommand("pause").Run()
}
// TODO: pass arguments
func (c *runcContainer) Checkpoint(runtime.Checkpoint) error {
return c.newCommand("checkpoint").Run()
}
// TODO: pass arguments
func (c *runcContainer) Restore(cp string) error {
return c.newCommand("restore").Run()
}
// TODO: pass arguments
func (c *runcContainer) DeleteCheckpoint(cp string) error {
return errors.New("not implemented")
}
// TODO: implement in runc
func (c *runcContainer) Checkpoints() ([]runtime.Checkpoint, error) {
return nil, errors.New("not implemented")
}
func (c *runcContainer) OOM() (<-chan struct{}, error) {
return nil, errors.New("not implemented")
}
func (c *runcContainer) newCommand(args ...string) *exec.Cmd {
cmd := exec.Command("runc", append([]string{"--root", c.stateDir, "--id", c.id}, args...)...)
cmd.Dir = c.path
return cmd
}
type runcProcess struct {
cmd *exec.Cmd
spec specs.Process
}
// pid of the container, not of runc
func (p *runcProcess) Pid() (int, error) {
return p.cmd.Process.Pid, nil
}
func (p *runcProcess) Spec() specs.Process {
return p.spec
}
func (p *runcProcess) Signal(s os.Signal) error {
return p.cmd.Process.Signal(s)
}
func (p *runcProcess) Close() error {
return nil
}
type runcRuntime struct {
stateDir string
}
func (r *runcRuntime) Type() string {
return "runc"
}
func (r *runcRuntime) Create(id, bundlePath, consolePath string) (runtime.Container, *runtime.IO, error) {
var s specs.Spec
f, err := os.Open(filepath.Join(bundlePath, "config.json"))
if err != nil {
return nil, nil, err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&s); err != nil {
return nil, nil, err
}
cmd := exec.Command("runc", "--root", r.stateDir, "--id", id, "start")
cmd.Dir = bundlePath
i, err := r.createIO(cmd)
if err != nil {
return nil, nil, err
}
return &runcContainer{
id: id,
path: bundlePath,
stateDir: r.stateDir,
initProcess: &runcProcess{
cmd: cmd,
spec: s.Process,
},
processes: make(map[int]*runcProcess),
}, i, nil
}
func (r *runcRuntime) createIO(cmd *exec.Cmd) (*runtime.IO, error) {
w, err := cmd.StdinPipe()
if err != nil {
return nil, err
}
ro, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
re, err := cmd.StderrPipe()
if err != nil {
return nil, err
}
return &runtime.IO{
Stdin: w,
Stdout: ro,
Stderr: re,
}, nil
}
func (r *runcRuntime) StartProcess(ci runtime.Container, p specs.Process, consolePath string) (runtime.Process, *runtime.IO, error) {
c, ok := ci.(*runcContainer)
if !ok {
return nil, nil, runtime.ErrInvalidContainerType
}
f, err := ioutil.TempFile("", "containerd")
if err != nil {
return nil, nil, err
}
err = json.NewEncoder(f).Encode(p)
f.Close()
if err != nil {
return nil, nil, err
}
cmd := c.newCommand("exec", f.Name())
i, err := r.createIO(cmd)
if err != nil {
return nil, nil, err
}
process := &runcProcess{
cmd: cmd,
spec: p,
}
if err := cmd.Start(); err != nil {
return nil, nil, err
}
pid, err := process.Pid()
if err != nil {
return nil, nil, err
}
c.processes[pid] = process
return process, i, nil
}

46
runtime/container.go
View File

@ -10,9 +10,27 @@ import (
type Process interface {
io.Closer
Pid() (int, error)
// ID of the process.
// This is either "init" when it is the container's init process or
// it is a user provided id for the process similar to the container id
ID() string
// Stdin returns the path the the processes stdin fifo
Stdin() string
// Stdout returns the path the the processes stdout fifo
Stdout() string
// Stderr returns the path the the processes stderr fifo
Stderr() string
// ExitFD returns the fd the provides an event when the process exits
ExitFD() int
// ExitStatus returns the exit status of the process or an error if it
// has not exited
ExitStatus() (int, error)
Spec() specs.Process
// Signal sends the provided signal to the process
Signal(os.Signal) error
// Container returns the container that the process belongs to
Container() Container
}
type State string
@ -77,20 +95,16 @@ type Checkpoint struct {
type Container interface {
// ID returns the container ID
ID() string
// Start starts the init process of the container
Start() error
// Path returns the path to the bundle
Path() string
// Pid returns the container's init process id
Pid() (int, error)
// SetExited sets the exit status of the container after its init dies
SetExited(status int)
// Delete deletes the container
// Start starts the init process of the container
Start() (Process, error)
// Delete removes the container's state and any resources
Delete() error
// Pid returns the container's init process id
// Pid() (int, error)
// Processes returns all the containers processes that have been added
Processes() ([]Process, error)
// RemoveProcess removes a specific process for the container because it exited
RemoveProcess(pid int) error
// State returns the containers runtime state
State() State
// Resume resumes a paused container
@ -98,15 +112,15 @@ type Container interface {
// Pause pauses a running container
Pause() error
// Checkpoints returns all the checkpoints for a container
Checkpoints() ([]Checkpoint, error)
// Checkpoints() ([]Checkpoint, error)
// Checkpoint creates a new checkpoint
Checkpoint(Checkpoint) error
// Checkpoint(Checkpoint) error
// DeleteCheckpoint deletes the checkpoint for the provided name
DeleteCheckpoint(name string) error
// DeleteCheckpoint(name string) error
// Restore restores the container to that of the checkpoint provided by name
Restore(name string) error
// Restore(name string) error
// Stats returns realtime container stats and resource information
Stats() (*Stat, error)
// Stats() (*Stat, error)
// OOM signals the channel if the container received an OOM notification
OOM() (<-chan struct{}, error)
// OOM() (<-chan struct{}, error)
}

172
runtime/lib.go Normal file
View File

@ -0,0 +1,172 @@
package runtime
import (
"encoding/json"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/opencontainers/specs"
)
const (
ExitFile = "exit"
ExitStatusFile = "exitStatus"
StateFile = "state.json"
InitProcessID = "init"
)
type state struct {
Bundle string `json:"bundle"`
}
// New returns a new container
func New(root, id, bundle string) (Container, error) {
c := &container{
root: root,
id: id,
bundle: bundle,
processes: make(map[string]*process),
}
if err := os.Mkdir(filepath.Join(root, id), 0755); err != nil {
return nil, err
}
f, err := os.Create(filepath.Join(root, id, StateFile))
if err != nil {
return nil, err
}
defer f.Close()
if err := json.NewEncoder(f).Encode(state{
Bundle: bundle,
}); err != nil {
return nil, err
}
return c, nil
}
func Load(root, id string) (Container, error) {
var s state
f, err := os.Open(filepath.Join(root, id, StateFile))
if err != nil {
return nil, err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&s); err != nil {
return nil, err
}
c := &container{
root: root,
id: id,
bundle: s.Bundle,
processes: make(map[string]*process),
}
dirs, err := ioutil.ReadDir(filepath.Join(root, id))
if err != nil {
return nil, err
}
for _, d := range dirs {
if !d.IsDir() {
continue
}
pid := d.Name()
// TODO: get the process spec from a state file in the process dir
p, err := loadProcess(filepath.Join(root, id, pid), pid, c, specs.Process{})
if err != nil {
if err == ErrProcessExited {
logrus.WithField("id", id).WithField("pid", pid).Debug("containerd: process exited while away")
// TODO: fire events to do the removal
if err := os.RemoveAll(filepath.Join(root, id, pid)); err != nil {
logrus.WithField("error", err).Warn("containerd: remove process state")
}
continue
}
return nil, err
}
c.processes[pid] = p
}
if len(c.processes) == 0 {
return nil, ErrContainerExited
}
return c, nil
}
type container struct {
// path to store runtime state information
root string
id string
bundle string
processes map[string]*process
}
func (c *container) ID() string {
return c.id
}
func (c *container) Path() string {
return c.bundle
}
func (c *container) Start() (Process, error) {
processRoot := filepath.Join(c.root, c.id, InitProcessID)
if err := os.MkdirAll(processRoot, 0755); err != nil {
return nil, err
}
cmd := exec.Command("containerd-shim", processRoot, c.id)
cmd.Dir = c.bundle
cmd.SysProcAttr = &syscall.SysProcAttr{
Setpgid: true,
}
spec, err := c.readSpec()
if err != nil {
return nil, err
}
p, err := newProcess(processRoot, InitProcessID, c, spec.Process)
if err != nil {
return nil, err
}
if err := cmd.Start(); err != nil {
return nil, err
}
c.processes[InitProcessID] = p
return p, nil
}
func (c *container) readSpec() (*specs.LinuxSpec, error) {
var spec specs.LinuxSpec
f, err := os.Open(filepath.Join(c.bundle, "config.json"))
if err != nil {
return nil, err
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&spec); err != nil {
return nil, err
}
return &spec, nil
}
func (c *container) Pause() error {
return errNotImplemented
}
func (c *container) Resume() error {
return errNotImplemented
}
func (c *container) State() State {
return Running
}
func (c *container) Delete() error {
return os.RemoveAll(filepath.Join(c.root, c.id))
}
func (c *container) Processes() ([]Process, error) {
out := []Process{}
for _, p := range c.processes {
out = append(out, p)
}
return out, nil
}

141
runtime/process.go Normal file
View File

@ -0,0 +1,141 @@
package runtime
import (
"io/ioutil"
"os"
"path/filepath"
"strconv"
"syscall"
"github.com/opencontainers/specs"
)
func newProcess(root, id string, c *container, s specs.Process) (*process, error) {
p := &process{
root: root,
id: id,
container: c,
spec: s,
}
// create fifo's for the process
for name, fd := range map[string]*string{
"stdin": &p.stdin,
"stdout": &p.stdout,
"stderr": &p.stderr,
} {
path := filepath.Join(root, name)
if err := syscall.Mkfifo(path, 0755); err != nil && !os.IsExist(err) {
return nil, err
}
*fd = path
}
exit, err := getExitPipe(filepath.Join(root, ExitFile))
if err != nil {
return nil, err
}
p.exitPipe = exit
return p, nil
}
func loadProcess(root, id string, c *container, s specs.Process) (*process, error) {
p := &process{
root: root,
id: id,
container: c,
spec: s,
stdin: filepath.Join(root, "stdin"),
stdout: filepath.Join(root, "stdout"),
stderr: filepath.Join(root, "stderr"),
}
if _, err := p.ExitStatus(); err != nil {
if err == ErrProcessNotExited {
exit, err := getExitPipe(filepath.Join(root, ExitFile))
if err != nil {
return nil, err
}
p.exitPipe = exit
return p, nil
}
return nil, err
}
return nil, ErrProcessExited
}
func getExitPipe(path string) (*os.File, error) {
if err := syscall.Mkfifo(path, 0755); err != nil && !os.IsExist(err) {
return nil, err
}
// add NONBLOCK in case the other side has already closed or else
// this function would never return
return os.OpenFile(path, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
}
type process struct {
root string
id string
// stdio fifos
stdin string
stdout string
stderr string
exitPipe *os.File
container *container
spec specs.Process
}
func (p *process) ID() string {
return p.id
}
func (p *process) Container() Container {
return p.container
}
// ExitFD returns the fd of the exit pipe
func (p *process) ExitFD() int {
return int(p.exitPipe.Fd())
}
func (p *process) ExitStatus() (int, error) {
data, err := ioutil.ReadFile(filepath.Join(p.root, ExitStatusFile))
if err != nil {
if os.IsNotExist(err) {
return -1, ErrProcessNotExited
}
return -1, err
}
if len(data) == 0 {
return -1, ErrProcessNotExited
}
i, err := strconv.Atoi(string(data))
if err != nil {
return -1, err
}
return i, nil
}
// Signal sends the provided signal to the process
func (p *process) Signal(s os.Signal) error {
return errNotImplemented
}
func (p *process) Spec() specs.Process {
return p.spec
}
func (p *process) Stdin() string {
return p.stdin
}
func (p *process) Stdout() string {
return p.stdout
}
func (p *process) Stderr() string {
return p.stderr
}
// Close closes any open files and/or resouces on the process
func (p *process) Close() error {
return p.exitPipe.Close()
}

20
runtime/runtime.go
View File

@ -1,10 +1,6 @@
package runtime
import (
"errors"
"github.com/opencontainers/specs"
)
import "errors"
var (
ErrNotChildProcess = errors.New("containerd: not a child process for container")
@ -13,14 +9,8 @@ var (
ErrCheckpointExists = errors.New("containerd: checkpoint already exists")
ErrContainerExited = errors.New("containerd: container has exited")
ErrTerminalsNotSupported = errors.New("containerd: terminals are not supported for runtime")
)
ErrProcessNotExited = errors.New("containerd: process has not exited")
ErrProcessExited = errors.New("containerd: process has exited")
// Runtime handles containers, containers handle their own actions
type Runtime interface {
// Type of the runtime
Type() string
// Create creates a new container initialized but without it starting it
Create(id, bundlePath, consolePath string) (Container, *IO, error)
// StartProcess adds a new process to the container
StartProcess(c Container, p specs.Process, consolePath string) (Process, *IO, error)
}
errNotImplemented = errors.New("containerd: not implemented")
)

58
supervisor/add_process.go
View File

@ -1,11 +1,5 @@
package supervisor
import (
"time"
"github.com/Sirupsen/logrus"
)
type AddProcessEvent struct {
s *Supervisor
}
@ -13,30 +7,32 @@ type AddProcessEvent struct {
// TODO: add this to worker for concurrent starts??? maybe not because of races where the container
// could be stopped and removed...
func (h *AddProcessEvent) Handle(e *Event) error {
start := time.Now()
ci, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
p, io, err := h.s.runtime.StartProcess(ci.container, *e.Process, e.Console)
if err != nil {
return err
}
if e.Pid, err = p.Pid(); err != nil {
return err
}
h.s.processes[e.Pid] = &containerInfo{
container: ci.container,
}
l, err := h.s.copyIO(e.Stdin, e.Stdout, e.Stderr, io)
if err != nil {
// log the error but continue with the other commands
logrus.WithFields(logrus.Fields{
"error": err,
"id": e.ID,
}).Error("log stdio")
}
h.s.processes[e.Pid].copier = l
ExecProcessTimer.UpdateSince(start)
/*
start := time.Now()
ci, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
p, io, err := h.s.runtime.StartProcess(ci.container, *e.Process, e.Console)
if err != nil {
return err
}
if e.Pid, err = p.Pid(); err != nil {
return err
}
h.s.processes[e.Pid] = &containerInfo{
container: ci.container,
}
l, err := h.s.copyIO(e.Stdin, e.Stdout, e.Stderr, io)
if err != nil {
// log the error but continue with the other commands
logrus.WithFields(logrus.Fields{
"error": err,
"id": e.ID,
}).Error("log stdio")
}
h.s.processes[e.Pid].copier = l
ExecProcessTimer.UpdateSince(start)
*/
return nil
}

26
supervisor/checkpoint.go
View File

@ -5,11 +5,14 @@ type CreateCheckpointEvent struct {
}
func (h *CreateCheckpointEvent) Handle(e *Event) error {
i, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
return i.container.Checkpoint(*e.Checkpoint)
/*
i, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
*/
return nil
// return i.container.Checkpoint(*e.Checkpoint)
}
type DeleteCheckpointEvent struct {
@ -17,9 +20,12 @@ type DeleteCheckpointEvent struct {
}
func (h *DeleteCheckpointEvent) Handle(e *Event) error {
i, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
return i.container.DeleteCheckpoint(e.Checkpoint.Name)
/*
i, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
*/
return nil
// return i.container.DeleteCheckpoint(e.Checkpoint.Name)
}

13
supervisor/create.go
View File

@ -1,6 +1,10 @@
package supervisor
import "time"
import (
"time"
"github.com/docker/containerd/runtime"
)
type StartEvent struct {
s *Supervisor
@ -8,22 +12,17 @@ type StartEvent struct {
func (h *StartEvent) Handle(e *Event) error {
start := time.Now()
container, io, err := h.s.runtime.Create(e.ID, e.BundlePath, e.Console)
container, err := runtime.New(h.s.stateDir, e.ID, e.BundlePath)
if err != nil {
return err
}
h.s.containerGroup.Add(1)
h.s.containers[e.ID] = &containerInfo{
container: container,
}
ContainersCounter.Inc(1)
task := &StartTask{
Err: e.Err,
IO: io,
Container: container,
Stdin: e.Stdin,
Stdout: e.Stdout,
Stderr: e.Stderr,
StartResponse: e.StartResponse,
}
if e.Checkpoint != nil {

1
supervisor/delete.go
View File

@ -29,7 +29,6 @@ func (h *DeleteEvent) Handle(e *Event) error {
Pid: e.Pid,
})
ContainersCounter.Dec(1)
h.s.containerGroup.Done()
ContainerDeleteTimer.UpdateSince(start)
}
return nil

9
supervisor/event.go
View File

@ -36,7 +36,9 @@ func NewEvent(t EventType) *Event {
}
type StartResponse struct {
Pid int
Stdin string
Stdout string
Stderr string
}
type Event struct {
@ -48,11 +50,12 @@ type Event struct {
Stderr string
Stdin string
Console string
Pid int
Pid string
Status int
Signal os.Signal
Process *specs.Process
Process runtime.Process
State runtime.State
ProcessSpec *specs.Process
Containers []runtime.Container
Checkpoint *runtime.Checkpoint
Err chan error

59
supervisor/exit.go
View File

@ -4,6 +4,7 @@ import (
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/containerd/runtime"
)
type ExitEvent struct {
@ -12,36 +13,36 @@ type ExitEvent struct {
func (h *ExitEvent) Handle(e *Event) error {
start := time.Now()
logrus.WithFields(logrus.Fields{"pid": e.Pid, "status": e.Status}).
Debug("containerd: process exited")
// is it the child process of a container
if info, ok := h.s.processes[e.Pid]; ok {
ne := NewEvent(ExecExitEventType)
ne.ID = info.container.ID()
ne.Pid = e.Pid
ne.Status = e.Status
h.s.SendEvent(ne)
return nil
}
// is it the main container's process
container, err := h.s.getContainerForPid(e.Pid)
proc := e.Process
status, err := proc.ExitStatus()
if err != nil {
if err != errNoContainerForPid {
logrus.WithField("error", err).Error("containerd: find containers main pid")
}
logrus.WithField("error", err).Error("containerd: get exit status")
}
logrus.WithFields(logrus.Fields{"pid": proc.ID(), "status": status}).Debug("containerd: process exited")
// if the process is the the init process of the container then
// fire a separate event for this process
if proc.ID() != runtime.InitProcessID {
ne := NewEvent(ExecExitEventType)
ne.ID = proc.Container().ID()
ne.Status = status
h.s.SendEvent(ne)
return nil
}
container.SetExited(e.Status)
container := proc.Container()
ne := NewEvent(DeleteEventType)
ne.ID = container.ID()
ne.Pid = e.Pid
ne.Status = e.Status
ne.Status = status
ne.Pid = proc.ID()
h.s.SendEvent(ne)
// remove stats collection for container
stopCollect := NewEvent(StopStatsEventType)
stopCollect.ID = container.ID()
h.s.SendEvent(stopCollect)
ExitProcessTimer.UpdateSince(start)
return nil
}
@ -51,14 +52,16 @@ type ExecExitEvent struct {
func (h *ExecExitEvent) Handle(e *Event) error {
// exec process: we remove this process without notifying the main event loop
info := h.s.processes[e.Pid]
if err := info.container.RemoveProcess(e.Pid); err != nil {
logrus.WithField("error", err).Error("containerd: find container for pid")
}
if err := info.copier.Close(); err != nil {
logrus.WithField("error", err).Error("containerd: close process IO")
}
delete(h.s.processes, e.Pid)
h.s.notifySubscribers(e)
/*
info := h.s.processes[e.Pid]
if err := info.container.RemoveProcess(e.Pid); err != nil {
logrus.WithField("error", err).Error("containerd: find container for pid")
}
if err := info.copier.Close(); err != nil {
logrus.WithField("error", err).Error("containerd: close process IO")
}
delete(h.s.processes, e.Pid)
h.s.notifySubscribers(e)
*/
return nil
}

7
supervisor/machine.go
View File

@ -3,15 +3,12 @@ package supervisor
import "github.com/cloudfoundry/gosigar"
type Machine struct {
ID string
Cpus int
Memory int64
}
func CollectMachineInformation(id string) (Machine, error) {
m := Machine{
ID: id,
}
func CollectMachineInformation() (Machine, error) {
m := Machine{}
cpu := sigar.CpuList{}
if err := cpu.Get(); err != nil {
return m, err

86
supervisor/monitor.go Normal file
View File

@ -0,0 +1,86 @@
package supervisor
import (
"sync"
"syscall"
"github.com/Sirupsen/logrus"
"github.com/docker/containerd/runtime"
)
func NewMonitor() (*Monitor, error) {
m := &Monitor{
processes: make(map[int]runtime.Process),
exits: make(chan runtime.Process, 1024),
}
fd, err := syscall.EpollCreate1(0)
if err != nil {
return nil, err
}
m.epollFd = fd
go m.start()
return m, nil
}
type Monitor struct {
m sync.Mutex
processes map[int]runtime.Process
exits chan runtime.Process
epollFd int
}
func (m *Monitor) Exits() chan runtime.Process {
return m.exits
}
func (m *Monitor) Monitor(p runtime.Process) error {
m.m.Lock()
defer m.m.Unlock()
fd := p.ExitFD()
event := syscall.EpollEvent{
Fd: int32(fd),
Events: syscall.EPOLLHUP,
}
if err := syscall.EpollCtl(m.epollFd, syscall.EPOLL_CTL_ADD, fd, &event); err != nil {
return err
}
m.processes[fd] = p
return nil
}
func (m *Monitor) Close() error {
return syscall.Close(m.epollFd)
}
func (m *Monitor) start() {
var events [128]syscall.EpollEvent
for {
n, err := syscall.EpollWait(m.epollFd, events[:], -1)
if err != nil {
if err == syscall.EINTR {
continue
}
logrus.WithField("error", err).Fatal("containerd: epoll wait")
}
// process events
for i := 0; i < n; i++ {
if events[i].Events == syscall.EPOLLHUP {
fd := int(events[i].Fd)
m.m.Lock()
proc := m.processes[fd]
delete(m.processes, fd)
if err = syscall.EpollCtl(m.epollFd, syscall.EPOLL_CTL_DEL, fd, &syscall.EpollEvent{
Events: syscall.EPOLLHUP,
Fd: int32(fd),
}); err != nil {
logrus.WithField("error", err).Fatal("containerd: epoll remove fd")
}
if err := proc.Close(); err != nil {
logrus.WithField("error", err).Error("containerd: close process IO")
}
m.m.Unlock()
m.exits <- proc
}
}
}
}

26
supervisor/signal.go
View File

@ -5,18 +5,20 @@ type SignalEvent struct {
}
func (h *SignalEvent) Handle(e *Event) error {
i, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
processes, err := i.container.Processes()
if err != nil {
return err
}
for _, p := range processes {
if pid, err := p.Pid(); err == nil && pid == e.Pid {
return p.Signal(e.Signal)
/*
i, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
}
processes, err := i.container.Processes()
if err != nil {
return err
}
for _, p := range processes {
if pid, err := p.Pid(); err == nil && pid == e.Pid {
return p.Signal(e.Signal)
}
}
*/
return ErrProcessNotFound
}

19
supervisor/stats_collector.go
View File

@ -9,7 +9,6 @@ import (
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/containerd/api/grpc/types"
"github.com/docker/containerd/runtime"
"github.com/docker/docker/pkg/pubsub"
@ -179,15 +178,17 @@ func (s *statsCollector) run() {
continue
}
for _, pair := range pairs {
stats, err := pair.ct.Stats()
if err != nil {
logrus.Errorf("Error getting stats for container ID %s", pair.ct.ID())
continue
}
/*
for _, pair := range pairs {
stats, err := pair.ct.Stats()
if err != nil {
logrus.Errorf("Error getting stats for container ID %s", pair.ct.ID())
continue
}
pair.pub.Publish(convertToPb(stats))
}
pair.pub.Publish(convertToPb(stats))
}
*/
}
}

142
supervisor/supervisor.go
View File

@ -1,18 +1,16 @@
package supervisor
import (
"io/ioutil"
"os"
"os/signal"
"path/filepath"
"sync"
"syscall"
"time"
"github.com/Sirupsen/logrus"
"github.com/docker/containerd/chanotify"
"github.com/docker/containerd/eventloop"
"github.com/docker/containerd/runtime"
"github.com/opencontainers/runc/libcontainer"
)
const (
@ -21,29 +19,27 @@ const (
)
// New returns an initialized Process supervisor.
func New(id, stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error) {
func New(stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error) {
if err := os.MkdirAll(stateDir, 0755); err != nil {
return nil, err
}
// register counters
r, err := newRuntime(filepath.Join(stateDir, id))
machine, err := CollectMachineInformation()
if err != nil {
return nil, err
}
machine, err := CollectMachineInformation(id)
monitor, err := NewMonitor()
if err != nil {
return nil, err
}
s := &Supervisor{
stateDir: stateDir,
containers: make(map[string]*containerInfo),
processes: make(map[int]*containerInfo),
runtime: r,
tasks: tasks,
machine: machine,
subscribers: make(map[chan *Event]struct{}),
statsCollector: newStatsCollector(statsInterval),
el: eventloop.NewChanLoop(defaultBufferSize),
monitor: monitor,
}
if oom {
s.notifier = chanotify.New()
@ -71,7 +67,10 @@ func New(id, stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, err
UnsubscribeStatsEventType: &UnsubscribeStatsEvent{s},
StopStatsEventType: &StopStatsEvent{s},
}
// start the container workers for concurrent container starts
go s.exitHandler()
if err := s.restore(); err != nil {
return nil, err
}
return s, nil
}
@ -84,9 +83,7 @@ type Supervisor struct {
// stateDir is the directory on the system to store container runtime state information.
stateDir string
containers map[string]*containerInfo
processes map[int]*containerInfo
handlers map[EventType]Handler
runtime runtime.Runtime
events chan *Event
tasks chan *StartTask
// we need a lock around the subscribers map only because additions and deletions from
@ -94,51 +91,18 @@ type Supervisor struct {
subscriberLock sync.RWMutex
subscribers map[chan *Event]struct{}
machine Machine
containerGroup sync.WaitGroup
statsCollector *statsCollector
notifier *chanotify.Notifier
el eventloop.EventLoop
monitor *Monitor
}
// Stop closes all tasks and sends a SIGTERM to each container's pid1 then waits for they to
// terminate. After it has handled all the SIGCHILD events it will close the signals chan
// and exit. Stop is a non-blocking call and will return after the containers have been signaled
func (s *Supervisor) Stop(sig chan os.Signal) {
func (s *Supervisor) Stop() {
// Close the tasks channel so that no new containers get started
close(s.tasks)
// send a SIGTERM to all containers
for id, i := range s.containers {
c := i.container
logrus.WithField("id", id).Debug("sending TERM to container processes")
procs, err := c.Processes()
if err != nil {
logrus.WithField("id", id).Warn("get container processes")
continue
}
if len(procs) == 0 {
continue
}
mainProc := procs[0]
if err := mainProc.Signal(syscall.SIGTERM); err != nil {
pid, _ := mainProc.Pid()
logrus.WithFields(logrus.Fields{
"id": id,
"pid": pid,
"error": err,
}).Error("send SIGTERM to process")
}
}
go func() {
logrus.Debug("waiting for containers to exit")
s.containerGroup.Wait()
logrus.Debug("all containers exited")
if s.notifier != nil {
s.notifier.Close()
}
// stop receiving signals and close the channel
signal.Stop(sig)
close(sig)
}()
}
// Close closes any open files in the supervisor but expects that Stop has been
@ -190,7 +154,6 @@ func (s *Supervisor) notifySubscribers(e *Event) {
// state of the Supervisor
func (s *Supervisor) Start() error {
logrus.WithFields(logrus.Fields{
"runtime": s.runtime.Type(),
"stateDir": s.stateDir,
}).Debug("Supervisor started")
return s.el.Start()
@ -202,45 +165,60 @@ func (s *Supervisor) Machine() Machine {
return s.machine
}
// getContainerForPid returns the container where the provided pid is the pid1 or main
// process in the container
func (s *Supervisor) getContainerForPid(pid int) (runtime.Container, error) {
for _, i := range s.containers {
container := i.container
cpid, err := container.Pid()
if err != nil {
if lerr, ok := err.(libcontainer.Error); ok {
if lerr.Code() == libcontainer.ProcessNotExecuted {
continue
}
}
logrus.WithField("error", err).Error("containerd: get container pid")
}
if pid == cpid {
return container, nil
}
}
return nil, errNoContainerForPid
}
// SendEvent sends the provided event the the supervisors main event loop
func (s *Supervisor) SendEvent(evt *Event) {
EventsCounter.Inc(1)
s.el.Send(&commonEvent{data: evt, sv: s})
}
func (s *Supervisor) copyIO(stdin, stdout, stderr string, i *runtime.IO) (*copier, error) {
config := &ioConfig{
Stdin: i.Stdin,
Stdout: i.Stdout,
Stderr: i.Stderr,
StdoutPath: stdout,
StderrPath: stderr,
StdinPath: stdin,
func (s *Supervisor) exitHandler() {
for p := range s.monitor.Exits() {
e := NewEvent(ExitEventType)
e.Process = p
s.SendEvent(e)
}
l, err := newCopier(config)
if err != nil {
return nil, err
}
return l, nil
}
func (s *Supervisor) monitorProcess(p runtime.Process) error {
return s.monitor.Monitor(p)
}
func (s *Supervisor) restore() error {
dirs, err := ioutil.ReadDir(s.stateDir)
if err != nil {
return err
}
for _, d := range dirs {
if !d.IsDir() {
continue
}
id := d.Name()
container, err := runtime.Load(s.stateDir, id)
if err != nil {
if err == runtime.ErrContainerExited {
logrus.WithField("id", id).Debug("containerd: container exited while away")
// TODO: fire events to do the removal
if err := os.RemoveAll(filepath.Join(s.stateDir, id)); err != nil {
logrus.WithField("error", err).Warn("containerd: remove container state")
}
continue
}
return err
}
processes, err := container.Processes()
if err != nil {
return err
}
ContainersCounter.Inc(1)
s.containers[id] = &containerInfo{
container: container,
}
logrus.WithField("id", id).Debug("containerd: container restored")
for _, p := range processes {
if err := s.monitorProcess(p); err != nil {
return err
}
}
}
return nil
}

12
supervisor/supervisor_linux.go
View File

@ -1,12 +0,0 @@
// +build libcontainer
package supervisor
import (
"github.com/docker/containerd/linux"
"github.com/docker/containerd/runtime"
)
func newRuntime(stateDir string) (runtime.Runtime, error) {
return linux.NewRuntime(stateDir)
}

12
supervisor/supervisor_runc.go
View File

@ -1,12 +0,0 @@
// +build runc
package supervisor
import (
"github.com/docker/containerd/runc"
"github.com/docker/containerd/runtime"
)
func newRuntime(stateDir string) (runtime.Runtime, error) {
return runc.NewRuntime(stateDir)
}

13
supervisor/supervisor_unsupported.go
View File

@ -1,13 +0,0 @@
// +build !libcontainer,!runc
package supervisor
import (
"errors"
"github.com/docker/containerd/runtime"
)
func newRuntime(stateDir string) (runtime.Runtime, error) {
return nil, errors.New("unsupported platform")
}

20
supervisor/update.go
View File

@ -27,15 +27,17 @@ func (h *UpdateEvent) Handle(e *Event) error {
}
}
if e.Signal != nil {
// signal the pid1/main process of the container
processes, err := container.Processes()
if err != nil {
return err
}
if len(processes) == 0 {
return ErrProcessNotFound
}
return processes[0].Signal(e.Signal)
/*
// signal the pid1/main process of the container
processes, err := container.Processes()
if err != nil {
return err
}
if len(processes) == 0 {
return ErrProcessNotFound
}
return processes[0].Signal(e.Signal)
*/
}
return nil
}

61
supervisor/worker.go
View File

@ -38,26 +38,24 @@ type worker struct {
func (w *worker) Start() {
defer w.wg.Done()
for t := range w.s.tasks {
started := time.Now()
l, err := w.s.copyIO(t.Stdin, t.Stdout, t.Stderr, t.IO)
if err != nil {
evt := NewEvent(DeleteEventType)
evt.ID = t.Container.ID()
w.s.SendEvent(evt)
t.Err <- err
continue
}
w.s.containers[t.Container.ID()].copier = l
var (
err error
process runtime.Process
started = time.Now()
)
if t.Checkpoint != "" {
if err := t.Container.Restore(t.Checkpoint); err != nil {
evt := NewEvent(DeleteEventType)
evt.ID = t.Container.ID()
w.s.SendEvent(evt)
t.Err <- err
continue
}
/*
if err := t.Container.Restore(t.Checkpoint); err != nil {
evt := NewEvent(DeleteEventType)
evt.ID = t.Container.ID()
w.s.SendEvent(evt)
t.Err <- err
continue
}
*/
} else {
if err := t.Container.Start(); err != nil {
process, err = t.Container.Start()
if err != nil {
evt := NewEvent(DeleteEventType)
evt.ID = t.Container.ID()
w.s.SendEvent(evt)
@ -65,22 +63,25 @@ func (w *worker) Start() {
continue
}
}
pid, err := t.Container.Pid()
if err != nil {
logrus.WithField("error", err).Error("containerd: get container main pid")
}
if w.s.notifier != nil {
n, err := t.Container.OOM()
if err != nil {
logrus.WithField("error", err).Error("containerd: notify OOM events")
} else {
w.s.notifier.Add(t.Container.ID(), n)
}
/*
if w.s.notifier != nil {
n, err := t.Container.OOM()
if err != nil {
logrus.WithField("error", err).Error("containerd: notify OOM events")
} else {
w.s.notifier.Add(n, t.Container.ID())
}
}
*/
if err := w.s.monitorProcess(process); err != nil {
logrus.WithField("error", err).Error("containerd: add process to monitor")
}
ContainerStartTimer.UpdateSince(started)
t.Err <- nil
t.StartResponse <- StartResponse{
Pid: pid,
Stdin: process.Stdin(),
Stdout: process.Stdout(),
Stderr: process.Stderr(),
}
}
}

38
util/reaper.go Normal file
View File

@ -0,0 +1,38 @@
package util
import (
"syscall"
"github.com/opencontainers/runc/libcontainer/utils"
)
// Exit is the wait4 information from an exited process
type Exit struct {
Pid int
Status int
}
// Reap reaps all child processes for the calling process and returns their
// exit information
func Reap() (exits []Exit, err error) {
var (
ws syscall.WaitStatus
rus syscall.Rusage
)
for {
pid, err := syscall.Wait4(-1, &ws, syscall.WNOHANG, &rus)
if err != nil {
if err == syscall.ECHILD {
return exits, nil
}
return exits, err
}
if pid <= 0 {
return exits, nil
}
exits = append(exits, Exit{
Pid: pid,
Status: utils.ExitStatus(ws),
})
}
}