From b044ff0f290ffe53ae930492e021b3c13b9c8bfb Mon Sep 17 00:00:00 2001 From: John Howard Date: Thu, 25 Feb 2016 12:59:34 -0800 Subject: [PATCH] Refactor process.go for platform specific Signed-off-by: John Howard Move process sorter to new file Signed-off-by: Michael Crosby Sort containers by id This will not be the most accurate sorting but atleast the list will be consistent inbetween calls. Signed-off-by: Michael Crosby Allow runtime to be configurable via daemon start This allows people to pass an alternate name or location to the runtime binary to start containers. Signed-off-by: Michael Crosby Fix state output for containers Return the proper state/status for a container by checking if the pid is still alive. Also fix the cleanup handling in the shim to make sure containers are not left behind. Signed-off-by: Michael Crosby Properly wait for container start Signed-off-by: Michael Crosby --- api/grpc/server/server.go | 6 +- api/grpc/types/api.pb.go | 217 ++++++++++++++++------------------- api/grpc/types/api.proto | 1 + containerd-shim/main.go | 45 +++++--- containerd-shim/process.go | 18 ++- containerd/main.go | 10 +- ctr/container.go | 1 + ctr/sort.go | 27 +++++ runtime/container.go | 21 +++- runtime/container_linux.go | 47 +++++++- runtime/container_windows.go | 4 + runtime/process.go | 30 ++--- runtime/runtime.go | 14 ++- runtime/spec_linux.go | 6 +- supervisor/create.go | 2 +- supervisor/exit.go | 14 ++- supervisor/sort.go | 27 +++++ supervisor/supervisor.go | 33 ++---- supervisor/worker.go | 6 +- 19 files changed, 323 insertions(+), 206 deletions(-) create mode 100644 ctr/sort.go create mode 100644 supervisor/sort.go diff --git a/api/grpc/server/server.go b/api/grpc/server/server.go index 12c200a..25d807c 100644 --- a/api/grpc/server/server.go +++ b/api/grpc/server/server.go @@ -144,7 +144,8 @@ func createAPIContainer(c runtime.Container, getPids bool) (*types.Container, er procs = append(procs, appendToProcs) } var pids []int - if getPids { + state := c.State() + if getPids && (state == runtime.Running || state == runtime.Paused) { if pids, err = c.Pids(); err != nil { return nil, grpc.Errorf(codes.Internal, "get all pids for container") } @@ -154,8 +155,9 @@ func createAPIContainer(c runtime.Container, getPids bool) (*types.Container, er BundlePath: c.Path(), Processes: procs, Labels: c.Labels(), - Status: string(c.State()), + Status: string(state), Pids: toUint32(pids), + Runtime: c.Runtime(), }, nil } diff --git a/api/grpc/types/api.pb.go b/api/grpc/types/api.pb.go index 2d08b46..10f4c38 100644 --- a/api/grpc/types/api.pb.go +++ b/api/grpc/types/api.pb.go @@ -310,6 +310,7 @@ type Container struct { Status string `protobuf:"bytes,4,opt,name=status" json:"status,omitempty"` Labels []string `protobuf:"bytes,5,rep,name=labels" json:"labels,omitempty"` Pids []uint32 `protobuf:"varint,6,rep,name=pids" json:"pids,omitempty"` + Runtime string `protobuf:"bytes,7,opt,name=runtime" json:"runtime,omitempty"` } func (m *Container) Reset() { *m = Container{} } @@ -1096,122 +1097,102 @@ var _API_serviceDesc = grpc.ServiceDesc{ } var fileDescriptor0 = []byte{ - // 1862 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xc4, 0x18, 0x49, 0x6f, 0x1c, 0x4d, - 0xd5, 0xb3, 0xcf, 0xbc, 0x59, 0xec, 0xa9, 0x78, 0x19, 0xcf, 0xf7, 0x65, 0xa1, 0x05, 0x24, 0x6c, - 0x56, 0x70, 0x12, 0x11, 0x21, 0x81, 0x94, 0x38, 0x01, 0x42, 0x62, 0x70, 0xda, 0xb6, 0xb8, 0x20, - 0x8d, 0xda, 0xdd, 0xc5, 0x4c, 0xe3, 0x9e, 0xee, 0x4e, 0x77, 0x8d, 0x3d, 0xbe, 0x70, 0xe4, 0xc8, - 0x91, 0x0b, 0x12, 0x17, 0x6e, 0x48, 0xfc, 0x0d, 0xf8, 0x27, 0x9c, 0x39, 0xf0, 0x03, 0x78, 0xb5, - 0x74, 0x75, 0xf5, 0x2c, 0x76, 0x90, 0x40, 0xdf, 0xa5, 0x55, 0x6f, 0x5f, 0xea, 0xbd, 0xd7, 0x55, - 0x05, 0x2d, 0x27, 0xf6, 0x0f, 0xe2, 0x24, 0x62, 0x11, 0xa9, 0xb1, 0x9b, 0x98, 0xa6, 0xd6, 0xef, - 0x4b, 0xb0, 0x7d, 0x1e, 0x7b, 0x0e, 0xa3, 0x27, 0x49, 0xe4, 0xd2, 0x34, 0xb5, 0xe9, 0xa7, 0x19, - 0x4d, 0x19, 0xe9, 0x41, 0xd9, 0xf7, 0x06, 0xa5, 0x47, 0xa5, 0x27, 0x2d, 0x1b, 0x57, 0x64, 0x0b, - 0x2a, 0x31, 0x22, 0xca, 0x02, 0xc1, 0x97, 0xe4, 0x01, 0x80, 0x1b, 0x44, 0x29, 0x3d, 0x65, 0x9e, - 0x1f, 0x0e, 0x2a, 0x48, 0x68, 0xda, 0x06, 0x86, 0x6c, 0x43, 0xed, 0xda, 0xf7, 0xd8, 0x64, 0x50, - 0x45, 0x52, 0xd7, 0x96, 0x00, 0xd9, 0x85, 0xfa, 0x84, 0xfa, 0xe3, 0x09, 0x1b, 0xd4, 0x04, 0x5a, - 0x41, 0xd6, 0x1e, 0xec, 0x2c, 0xf8, 0x91, 0xc6, 0x51, 0x98, 0x52, 0xeb, 0xef, 0x25, 0xd8, 0x3d, - 0x4a, 0x28, 0x52, 0x8e, 0xa2, 0x90, 0x39, 0x7e, 0x48, 0x93, 0x75, 0x3e, 0xa2, 0x47, 0x17, 0xb3, - 0xd0, 0x0b, 0xe8, 0x89, 0x83, 0x66, 0xa5, 0xab, 0x06, 0x46, 0x78, 0x3c, 0xa1, 0xee, 0x65, 0x1c, - 0xf9, 0x21, 0x13, 0x1e, 0x23, 0x3d, 0xc7, 0x70, 0x8f, 0x53, 0x11, 0x4c, 0x55, 0x90, 0x24, 0xc0, - 0x3d, 0xc6, 0x45, 0x34, 0x93, 0x1e, 0xb7, 0x6c, 0x05, 0x29, 0x3c, 0x4d, 0x92, 0x41, 0x5d, 0xe3, - 0x11, 0xe2, 0xf8, 0xc0, 0xb9, 0xa0, 0x41, 0x3a, 0x68, 0x3c, 0xaa, 0x70, 0xbc, 0x84, 0xac, 0x77, - 0xb0, 0xb7, 0x14, 0x87, 0x8c, 0x91, 0x1c, 0x40, 0xcb, 0xcd, 0x90, 0x22, 0x9e, 0xf6, 0xe1, 0xd6, - 0x81, 0xd8, 0xa0, 0x83, 0x9c, 0x39, 0x67, 0x41, 0x55, 0xdd, 0x53, 0x7f, 0x1c, 0x3a, 0xc1, 0xe7, - 0xef, 0x16, 0xf7, 0x56, 0x88, 0x88, 0xb8, 0x31, 0xef, 0x12, 0xb2, 0xb6, 0xa0, 0x97, 0xa9, 0x52, - 0x09, 0xff, 0x57, 0x09, 0xfa, 0xaf, 0x3c, 0xef, 0x8e, 0x7a, 0x18, 0x42, 0x93, 0xd1, 0x64, 0xea, - 0x73, 0x8d, 0x65, 0xb1, 0xf7, 0x1a, 0x26, 0x0f, 0xa1, 0x3a, 0x4b, 0x31, 0x92, 0x8a, 0x88, 0xa4, - 0xad, 0x22, 0x39, 0x47, 0x94, 0x2d, 0x08, 0x84, 0x40, 0xd5, 0x49, 0xc6, 0x29, 0xe6, 0x99, 0x27, - 0x48, 0xac, 0xb9, 0xcb, 0x34, 0xbc, 0xc2, 0x1c, 0x73, 0x14, 0x5f, 0x72, 0x8c, 0x7b, 0xed, 0xa9, - 0xec, 0xf2, 0x65, 0x16, 0x56, 0x23, 0x0f, 0x4b, 0x6f, 0x59, 0x73, 0xf5, 0x96, 0xb5, 0xd6, 0x6c, - 0x19, 0x98, 0x5b, 0x66, 0xd9, 0x50, 0xe5, 0xde, 0x71, 0xfd, 0x33, 0x15, 0x65, 0xd7, 0xe6, 0x4b, - 0x8e, 0x19, 0xab, 0x44, 0x22, 0x06, 0x97, 0xe4, 0x9b, 0xd0, 0x73, 0x3c, 0xcf, 0x67, 0x7e, 0x84, - 0xa1, 0xfe, 0xd4, 0xf7, 0x52, 0x0c, 0xb3, 0x82, 0xc4, 0x05, 0xac, 0xb5, 0x0d, 0xc4, 0xcc, 0xa2, - 0x4a, 0xee, 0xaf, 0x75, 0x11, 0xe8, 0xb2, 0x5b, 0x97, 0xe1, 0xef, 0x17, 0xaa, 0xb5, 0x2c, 0x72, - 0xd9, 0xcf, 0xaa, 0x22, 0x97, 0x36, 0x98, 0xac, 0x21, 0x0c, 0x96, 0xb5, 0x2b, 0xcb, 0x3f, 0x82, - 0xbd, 0x37, 0x34, 0xa0, 0x9f, 0x63, 0x19, 0xb7, 0x27, 0x74, 0xa6, 0x54, 0x95, 0x8f, 0x58, 0x73, - 0xd5, 0xcb, 0xe2, 0x4a, 0xf5, 0x63, 0xd8, 0xf9, 0xe0, 0xa7, 0xec, 0x4e, 0xc5, 0xd6, 0xef, 0x00, - 0x72, 0x26, 0x6d, 0xa6, 0x94, 0x9b, 0xe1, 0x38, 0x3a, 0xf7, 0x99, 0x2a, 0x29, 0xb1, 0xe6, 0x7b, - 0xc0, 0xdc, 0x58, 0x4d, 0x18, 0xbe, 0x24, 0x8f, 0xa0, 0x3d, 0x0b, 0xfd, 0xf9, 0x69, 0xe4, 0x5e, - 0x52, 0x96, 0x8a, 0x76, 0x6d, 0xda, 0x26, 0x4a, 0xd4, 0xc5, 0x84, 0x06, 0x81, 0xe8, 0xd9, 0xa6, - 0x2d, 0x01, 0xeb, 0x18, 0x76, 0x17, 0x1d, 0x55, 0x1d, 0xf8, 0x0c, 0xda, 0x79, 0x1e, 0x53, 0x74, - 0xa9, 0xb2, 0x3a, 0xdb, 0x26, 0x97, 0xf5, 0x00, 0x3a, 0xa7, 0x0c, 0xb3, 0xbd, 0x2e, 0xdc, 0x27, - 0xd0, 0xd3, 0xed, 0x2b, 0x18, 0x65, 0x01, 0x3a, 0x6c, 0x96, 0x2a, 0x2e, 0x05, 0x59, 0xff, 0x2e, - 0x41, 0x43, 0x95, 0x4a, 0x56, 0xe4, 0xa5, 0xbc, 0xc8, 0xbf, 0x92, 0x5e, 0xfb, 0x12, 0x5a, 0xe9, - 0x4d, 0xca, 0xe8, 0xf4, 0x44, 0x75, 0x5c, 0xd7, 0xce, 0x11, 0xff, 0xa3, 0xbe, 0xfb, 0x5b, 0x09, - 0x5a, 0x3a, 0x43, 0xff, 0xf5, 0x38, 0xff, 0x2e, 0xb4, 0x62, 0x99, 0x33, 0x2a, 0x9b, 0xb0, 0x7d, - 0xd8, 0x53, 0xf1, 0x67, 0x6d, 0x97, 0x33, 0x18, 0xa9, 0xaf, 0x9a, 0xa9, 0x37, 0xc6, 0x75, 0xcd, - 0x1c, 0xd7, 0x3c, 0x6f, 0x31, 0xef, 0xee, 0xba, 0xe8, 0x6e, 0xb1, 0xb6, 0x5e, 0x40, 0xe3, 0xd8, - 0x71, 0x27, 0xe8, 0x2d, 0x27, 0xbb, 0xb1, 0xda, 0x47, 0x24, 0xf3, 0x35, 0x57, 0x35, 0xa5, 0xd3, - 0x28, 0xb9, 0x11, 0xce, 0x56, 0x6d, 0x05, 0x59, 0x97, 0x38, 0xae, 0x65, 0x9d, 0xa8, 0x6a, 0x7b, - 0x8a, 0xad, 0x9d, 0x85, 0x9d, 0x15, 0xdb, 0xf2, 0xc0, 0x37, 0x78, 0xc8, 0x13, 0x68, 0x4c, 0xa5, - 0x65, 0x35, 0x09, 0xb2, 0x48, 0x95, 0x3f, 0x76, 0x46, 0xc6, 0x59, 0xb6, 0x2b, 0x7f, 0xa4, 0x77, - 0xfe, 0x2e, 0x57, 0xff, 0x24, 0x64, 0x8e, 0x2a, 0x85, 0xf2, 0xdc, 0x87, 0xbd, 0x25, 0x9d, 0xaa, - 0xf7, 0xbf, 0x07, 0xdd, 0xb7, 0x57, 0x14, 0xbb, 0x21, 0xb3, 0x82, 0x75, 0xc3, 0xfc, 0x29, 0xae, - 0x9c, 0x69, 0x2c, 0x8c, 0x55, 0xed, 0x1c, 0x61, 0xa5, 0x50, 0x13, 0xec, 0x3c, 0x7f, 0x3c, 0x80, - 0xac, 0xf9, 0xf9, 0x5a, 0x39, 0x58, 0xd6, 0x0e, 0x16, 0xdd, 0xe9, 0xea, 0x2d, 0x53, 0x8e, 0x57, - 0x73, 0xc7, 0x0b, 0x46, 0x6b, 0x8b, 0x46, 0xff, 0x50, 0x86, 0xce, 0x2f, 0x28, 0xbb, 0x8e, 0x92, - 0x4b, 0xbe, 0x0f, 0xe9, 0xca, 0xc9, 0xb3, 0x0f, 0xcd, 0x64, 0x3e, 0xba, 0xb8, 0x61, 0x58, 0x4c, - 0x72, 0xfb, 0x1a, 0xc9, 0xfc, 0x35, 0x07, 0xc9, 0x7d, 0x00, 0x24, 0x9d, 0x38, 0x72, 0xda, 0x54, - 0xa4, 0xfa, 0x64, 0xae, 0x10, 0xe4, 0x0b, 0x68, 0xd9, 0xf3, 0x11, 0xd6, 0x73, 0x94, 0xc8, 0xe2, - 0xaa, 0xda, 0xa8, 0xea, 0xad, 0x80, 0xb9, 0x2c, 0x12, 0xbd, 0x24, 0x8a, 0x63, 0xea, 0x65, 0xae, - 0x25, 0xf3, 0x37, 0x12, 0xc1, 0xad, 0x9e, 0x65, 0x56, 0xeb, 0xd2, 0x2a, 0xcb, 0xad, 0x22, 0x29, - 0x56, 0x56, 0x1b, 0x2a, 0x28, 0xd3, 0xea, 0x99, 0xb6, 0xda, 0x94, 0x56, 0x99, 0x61, 0xf5, 0x2c, - 0xb7, 0xda, 0xca, 0x64, 0x95, 0x55, 0xeb, 0xaf, 0x25, 0x68, 0x1e, 0xc5, 0xb3, 0xf3, 0xd4, 0x19, - 0x53, 0x9c, 0x20, 0x6d, 0x16, 0x31, 0x27, 0x18, 0xcd, 0x38, 0xa8, 0xb6, 0x0c, 0x04, 0x4a, 0x32, - 0x7c, 0x0d, 0x3a, 0x31, 0x4d, 0xb0, 0xc2, 0x15, 0x47, 0x19, 0xeb, 0xb5, 0x6a, 0xb7, 0x25, 0x4e, - 0xb2, 0x1c, 0xc0, 0x3d, 0x41, 0x1b, 0xf9, 0xe1, 0xe8, 0x92, 0x26, 0x21, 0x0d, 0xa6, 0x91, 0x47, - 0x55, 0xaa, 0xfa, 0x82, 0xf4, 0x2e, 0x7c, 0xaf, 0x09, 0xe4, 0xdb, 0xd0, 0xd7, 0xfc, 0x7c, 0x4a, - 0x09, 0x6e, 0x99, 0xba, 0x4d, 0xc5, 0x7d, 0xae, 0xd0, 0xf8, 0xd3, 0xe8, 0x9d, 0x4d, 0xf0, 0xcc, - 0xca, 0x02, 0x3f, 0x1c, 0xbf, 0x71, 0x98, 0x43, 0x06, 0xd0, 0x40, 0xe3, 0x7e, 0xe4, 0xa5, 0xca, - 0xdb, 0x0c, 0x24, 0xdf, 0x81, 0x3e, 0x93, 0xbc, 0xd4, 0x1b, 0x65, 0x3c, 0x72, 0x37, 0xb7, 0x34, - 0xe1, 0x44, 0x31, 0x7f, 0x03, 0x7a, 0x39, 0x33, 0xaf, 0x16, 0xe5, 0x6f, 0x57, 0x63, 0xcf, 0x10, - 0x69, 0xfd, 0x49, 0x26, 0x4b, 0x56, 0x0e, 0xce, 0x9c, 0x3c, 0x11, 0xf2, 0xa4, 0xb6, 0x99, 0x35, - 0xae, 0x4a, 0x86, 0xdd, 0xd4, 0x69, 0xf9, 0x31, 0x6c, 0x32, 0xed, 0xfa, 0x08, 0x1b, 0xc8, 0x51, - 0xdd, 0xbb, 0xa3, 0x64, 0x8a, 0x81, 0xd9, 0x3d, 0x56, 0x0c, 0x14, 0x33, 0x2f, 0x47, 0xae, 0x32, - 0x28, 0xfd, 0x6b, 0x4b, 0x9c, 0x30, 0x61, 0x45, 0x00, 0xc7, 0x62, 0xca, 0x08, 0x01, 0x1c, 0xcb, - 0xe6, 0x2e, 0x4a, 0x80, 0x97, 0xca, 0xd4, 0x99, 0xeb, 0xdd, 0x13, 0xa5, 0x82, 0x08, 0xe9, 0x23, - 0x26, 0xf3, 0x37, 0x8e, 0x1f, 0xb8, 0xea, 0x44, 0x8c, 0xc9, 0x54, 0x20, 0x57, 0x16, 0xf8, 0x53, - 0xfc, 0x19, 0xcb, 0x8d, 0x91, 0x80, 0xf5, 0x97, 0x32, 0xb4, 0xa5, 0x45, 0x99, 0x11, 0xe4, 0x72, - 0x71, 0xf4, 0x68, 0x93, 0x02, 0x20, 0x8f, 0x33, 0x47, 0x8a, 0xe7, 0x96, 0xdc, 0xd5, 0xcc, 0x37, - 0x1c, 0x85, 0xe9, 0xb5, 0x13, 0x1b, 0x01, 0xae, 0xe4, 0x6e, 0x71, 0x26, 0xe9, 0xf0, 0x73, 0xe8, - 0xc8, 0x12, 0x53, 0x32, 0xd5, 0x75, 0x32, 0x6d, 0xc9, 0x26, 0xa5, 0x9e, 0xf1, 0x1f, 0x16, 0xfa, - 0x2b, 0xa6, 0x7c, 0xfb, 0xf0, 0x7e, 0x81, 0x5d, 0x44, 0x72, 0x20, 0xbe, 0x6f, 0x43, 0x96, 0xdc, - 0xd8, 0x92, 0x77, 0xf8, 0x12, 0x20, 0x47, 0xf2, 0xb1, 0x73, 0x49, 0x6f, 0xb2, 0x1f, 0x33, 0x2e, - 0x79, 0xec, 0x57, 0x4e, 0x30, 0xcb, 0x92, 0x2a, 0x81, 0x1f, 0x96, 0x5f, 0x96, 0x2c, 0x17, 0x36, - 0x5f, 0x07, 0x97, 0x7e, 0x64, 0x88, 0x23, 0xf3, 0xd4, 0xf9, 0x6d, 0x94, 0x64, 0x89, 0x12, 0x80, - 0xc0, 0xfa, 0x21, 0x62, 0x95, 0x0a, 0x01, 0xf0, 0x49, 0x18, 0xc5, 0x6a, 0x08, 0xe3, 0x2a, 0x37, - 0x54, 0x35, 0x0c, 0x59, 0xff, 0xac, 0x02, 0xe4, 0x56, 0xc8, 0x29, 0x0c, 0xfd, 0x68, 0x84, 0x6d, - 0x73, 0xe5, 0xbb, 0x54, 0xce, 0x94, 0x51, 0x42, 0xdd, 0x59, 0x92, 0xfa, 0x57, 0x54, 0xfd, 0x65, - 0x76, 0x55, 0xdc, 0x0b, 0xce, 0xd9, 0x7b, 0x08, 0x49, 0x41, 0x31, 0x7c, 0xec, 0x4c, 0x8c, 0xfc, - 0x1c, 0x76, 0x72, 0xa5, 0x9e, 0xa1, 0xaf, 0x7c, 0xab, 0xbe, 0x7b, 0x5a, 0x9f, 0x97, 0xeb, 0xfa, - 0x09, 0x20, 0x7a, 0x84, 0xbf, 0x89, 0x59, 0x41, 0x53, 0xe5, 0x56, 0x4d, 0x7d, 0x3f, 0xfa, 0x28, - 0x24, 0x72, 0x3d, 0x1f, 0x61, 0xdf, 0x08, 0x94, 0x77, 0xae, 0xa1, 0xad, 0x7a, 0xab, 0xb6, 0x5d, - 0xed, 0x17, 0xef, 0xed, 0x5c, 0xe5, 0x7b, 0x40, 0xca, 0xe8, 0xda, 0xf1, 0xd9, 0xa2, 0xbe, 0xda, - 0x5d, 0x71, 0xfe, 0x0a, 0x85, 0x8a, 0xca, 0x64, 0x9c, 0x53, 0x9a, 0x8c, 0x0b, 0x71, 0xd6, 0xef, - 0x8a, 0xf3, 0x58, 0x48, 0xe4, 0x7a, 0x5e, 0x03, 0x22, 0x17, 0xfd, 0x69, 0xdc, 0xaa, 0x65, 0xd3, - 0x8f, 0x8a, 0xbe, 0x1c, 0x41, 0x3f, 0xa5, 0x2e, 0xc3, 0x9f, 0x82, 0xa1, 0xa3, 0x79, 0xab, 0x8e, - 0x2d, 0x25, 0xa0, 0x95, 0x58, 0x9f, 0xa0, 0xf3, 0xb3, 0xd9, 0x98, 0xb2, 0xe0, 0x42, 0xf7, 0xfc, - 0xff, 0x7b, 0xcc, 0xfc, 0x03, 0xc7, 0xcc, 0xd1, 0x38, 0x89, 0x66, 0x71, 0x61, 0xf0, 0xca, 0x1e, - 0x5e, 0x1a, 0xbc, 0x82, 0x47, 0x0c, 0x5e, 0xc9, 0xfd, 0x02, 0x3a, 0xf2, 0xec, 0xa5, 0x04, 0xe4, - 0x14, 0x22, 0xcb, 0x4d, 0x6f, 0xb7, 0xa7, 0xc6, 0x2c, 0x3b, 0x84, 0xf6, 0x05, 0x4f, 0x86, 0x92, - 0x2a, 0x4e, 0xa3, 0x3c, 0x4d, 0x78, 0x0a, 0xcd, 0xbb, 0x0e, 0xef, 0xe2, 0x13, 0x99, 0x1b, 0x25, - 0x25, 0x0b, 0xf0, 0xeb, 0x99, 0x73, 0x79, 0x0c, 0x07, 0x66, 0x0e, 0x65, 0xaa, 0x3b, 0x13, 0x03, - 0x35, 0x3c, 0x83, 0xfe, 0x12, 0xcb, 0x8a, 0xa9, 0xf3, 0x2d, 0x73, 0xea, 0xb4, 0x0f, 0xef, 0x29, - 0x4b, 0xa6, 0xa8, 0x39, 0x8a, 0xfe, 0x5c, 0x92, 0xc7, 0x4f, 0x7d, 0x09, 0x25, 0x2f, 0xa1, 0x1b, - 0xca, 0xe3, 0x90, 0xce, 0x67, 0xc5, 0x50, 0x64, 0x1e, 0x95, 0xec, 0x4e, 0x68, 0x1e, 0x9c, 0x30, - 0xaf, 0xae, 0x08, 0x68, 0x65, 0x5e, 0x8d, 0x58, 0xf1, 0xa2, 0x64, 0x6c, 0x5e, 0xe1, 0x78, 0x56, - 0x59, 0x3c, 0x9e, 0xa9, 0x6b, 0xd4, 0xba, 0xa7, 0x86, 0xc3, 0x3f, 0xd6, 0xa1, 0xf2, 0xea, 0xe4, - 0x1d, 0xb1, 0x61, 0x73, 0xe1, 0x01, 0x85, 0x64, 0x63, 0x7c, 0xf5, 0x03, 0xd1, 0xf0, 0xc1, 0x3a, - 0xb2, 0x3a, 0xbc, 0x6e, 0x70, 0x9d, 0x0b, 0x27, 0x5b, 0xad, 0x73, 0xf5, 0x29, 0x5a, 0xeb, 0x5c, - 0x77, 0x20, 0xde, 0x20, 0x3f, 0x80, 0xba, 0x7c, 0x52, 0x21, 0xdb, 0x8a, 0xb7, 0xf0, 0x58, 0x33, - 0xdc, 0x59, 0xc0, 0x6a, 0xc1, 0x0f, 0xd0, 0x2d, 0xbc, 0x81, 0x91, 0x2f, 0x0a, 0xb6, 0x8a, 0x2f, - 0x32, 0xc3, 0x2f, 0x57, 0x13, 0xb5, 0xb6, 0x23, 0x80, 0xfc, 0x01, 0x82, 0x0c, 0x14, 0xf7, 0xd2, - 0xcb, 0xce, 0x70, 0x7f, 0x05, 0x45, 0x2b, 0x39, 0x87, 0xad, 0xc5, 0x17, 0x05, 0xb2, 0x90, 0xd5, - 0xc5, 0x5b, 0xff, 0xf0, 0xe1, 0x5a, 0xba, 0xa9, 0x76, 0xf1, 0x35, 0x41, 0xab, 0x5d, 0xf3, 0x4a, - 0xa1, 0xd5, 0xae, 0x7d, 0x86, 0xd8, 0x20, 0xbf, 0x84, 0x5e, 0xf1, 0x7e, 0x4f, 0xb2, 0x24, 0xad, - 0x7c, 0x9f, 0x18, 0xde, 0x5f, 0x43, 0xd5, 0x0a, 0x9f, 0x43, 0x4d, 0x5e, 0xdc, 0xb3, 0xde, 0x30, - 0xef, 0xfb, 0xc3, 0xed, 0x22, 0x52, 0x4b, 0x3d, 0x85, 0xba, 0xbc, 0x13, 0xe9, 0x02, 0x28, 0x5c, - 0x91, 0x86, 0x1d, 0x13, 0x6b, 0x6d, 0x3c, 0x2d, 0x65, 0x76, 0xd2, 0x82, 0x9d, 0x74, 0x95, 0x1d, - 0x63, 0x73, 0x2e, 0xea, 0xe2, 0x29, 0xf7, 0xd9, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x46, 0x74, - 0x2e, 0x01, 0xd7, 0x15, 0x00, 0x00, + // 1541 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xeb, 0x6e, 0x1b, 0x45, + 0x14, 0x8e, 0x2f, 0xb1, 0xe3, 0xe3, 0x4b, 0x92, 0xcd, 0xcd, 0x71, 0x29, 0x0d, 0xdb, 0x42, 0x2b, + 0x54, 0x45, 0x25, 0xe5, 0x52, 0x8a, 0x84, 0x28, 0x69, 0x45, 0x41, 0x6d, 0x09, 0x4d, 0x82, 0xc4, + 0x2f, 0x6b, 0xbd, 0x3b, 0xd8, 0x43, 0xd6, 0xbb, 0xcb, 0xce, 0x6c, 0x2e, 0xaf, 0x00, 0xe2, 0x2f, + 0x6f, 0x81, 0xc4, 0x2f, 0x1e, 0x80, 0xc7, 0xe1, 0x29, 0x38, 0x73, 0x5b, 0xef, 0xae, 0x2f, 0x01, + 0x21, 0xfe, 0x58, 0x9a, 0x99, 0x73, 0xf9, 0xce, 0x77, 0x2e, 0x3b, 0x63, 0x68, 0x38, 0x11, 0xdd, + 0x8f, 0xe2, 0x90, 0x87, 0xd6, 0x32, 0xbf, 0x8a, 0x08, 0xb3, 0x07, 0xb0, 0x79, 0x1a, 0x79, 0x0e, + 0x27, 0x47, 0x71, 0xe8, 0x12, 0xc6, 0x5e, 0x93, 0x1f, 0x13, 0xc2, 0xb8, 0x05, 0x50, 0xa6, 0x5e, + 0xb7, 0xb4, 0x57, 0xba, 0xd7, 0xb0, 0x9a, 0x50, 0x89, 0x70, 0x51, 0x96, 0x0b, 0x3c, 0x71, 0xfd, + 0x90, 0x91, 0x63, 0xee, 0xd1, 0xa0, 0x5b, 0xc1, 0xbd, 0x15, 0xab, 0x0d, 0xcb, 0x17, 0xd4, 0xe3, + 0xa3, 0x6e, 0x15, 0x97, 0x6d, 0xab, 0x03, 0xb5, 0x11, 0xa1, 0xc3, 0x11, 0xef, 0x2e, 0x8b, 0xb5, + 0xbd, 0x03, 0x5b, 0x05, 0x1f, 0x2c, 0x0a, 0x03, 0x46, 0xec, 0x9f, 0x4b, 0xb0, 0x7d, 0x18, 0x13, + 0x3c, 0x39, 0x0c, 0x03, 0xee, 0xd0, 0x80, 0xc4, 0xb3, 0xfc, 0xe3, 0x62, 0x90, 0x04, 0x9e, 0x4f, + 0x8e, 0x1c, 0xf4, 0x31, 0x81, 0x31, 0x22, 0xee, 0x59, 0x14, 0xd2, 0x80, 0x4b, 0x18, 0x0d, 0x01, + 0x83, 0x49, 0x54, 0x55, 0xb9, 0x44, 0x18, 0xb8, 0x0c, 0x13, 0x05, 0xc3, 0xac, 0x49, 0x1c, 0x77, + 0x6b, 0x66, 0xed, 0x3b, 0x03, 0xe2, 0xb3, 0x6e, 0x7d, 0xaf, 0x72, 0xaf, 0x61, 0x7f, 0x0a, 0x3b, + 0x53, 0x60, 0x14, 0x50, 0xeb, 0x36, 0x34, 0x5c, 0xb3, 0x29, 0x41, 0x35, 0x0f, 0xd6, 0xf6, 0x25, + 0x81, 0xfb, 0xa9, 0xb0, 0xfd, 0x08, 0xda, 0xc7, 0x74, 0x18, 0x38, 0xfe, 0xb5, 0x1c, 0x0a, 0x24, + 0x52, 0x52, 0x02, 0x6f, 0xdb, 0x6b, 0xd0, 0x31, 0x9a, 0x9a, 0x99, 0xdf, 0x4b, 0xb0, 0xfe, 0xc4, + 0xf3, 0x16, 0x24, 0x65, 0x0d, 0x56, 0x38, 0x89, 0xc7, 0x54, 0x58, 0x29, 0xcb, 0x2c, 0xec, 0x42, + 0x35, 0x61, 0x88, 0xaf, 0x22, 0xf1, 0x35, 0x35, 0xbe, 0x53, 0xdc, 0xb2, 0x5a, 0x50, 0x75, 0xe2, + 0x21, 0x43, 0x62, 0x2a, 0x0a, 0x0b, 0x09, 0xce, 0x91, 0x15, 0xbd, 0x70, 0x2f, 0x3c, 0x4d, 0x89, + 0x46, 0x59, 0xcf, 0xd3, 0xb9, 0x52, 0xa0, 0xb3, 0x51, 0xa0, 0x13, 0xc4, 0x1a, 0xc3, 0xaf, 0x4a, + 0x5f, 0x68, 0x23, 0xd1, 0x28, 0xdb, 0x62, 0x31, 0xd4, 0x61, 0xb7, 0xad, 0x6d, 0xe8, 0x38, 0x9e, + 0x47, 0x39, 0x0d, 0x11, 0xf4, 0x17, 0xd4, 0x63, 0x08, 0xb5, 0x82, 0xe1, 0x6f, 0x82, 0x95, 0x8d, + 0x55, 0x53, 0xf0, 0x22, 0x4d, 0x47, 0x9a, 0xe7, 0x59, 0x3c, 0xbc, 0x9d, 0x2b, 0x84, 0xb2, 0x8c, + 0x7d, 0xdd, 0xe4, 0x26, 0x3d, 0xb0, 0x7b, 0xd0, 0x9d, 0xb6, 0xa6, 0x3d, 0x3d, 0x84, 0x9d, 0xa7, + 0xc4, 0x27, 0xd7, 0x79, 0x42, 0x12, 0x03, 0x67, 0x4c, 0x54, 0x0e, 0x85, 0xc1, 0x69, 0x25, 0x6d, + 0xf0, 0x36, 0x6c, 0xbd, 0xa0, 0x8c, 0x2f, 0x34, 0x67, 0x7f, 0x07, 0x30, 0x11, 0x48, 0x8d, 0xa7, + 0xae, 0xc8, 0x25, 0xe5, 0x3a, 0xb1, 0x48, 0x22, 0x77, 0x23, 0xdd, 0x6b, 0x1b, 0xd0, 0x4c, 0x02, + 0x7a, 0x79, 0x1c, 0xba, 0x67, 0x84, 0x33, 0x59, 0xea, 0xb2, 0x01, 0xd9, 0x88, 0xf8, 0xbe, 0xac, + 0xf4, 0x15, 0xfb, 0x33, 0xd8, 0x2e, 0xfa, 0xd7, 0x85, 0xfc, 0x0e, 0x34, 0x27, 0x6c, 0x31, 0xf4, + 0x56, 0x99, 0x47, 0x57, 0xeb, 0x98, 0x23, 0x5b, 0xb3, 0x80, 0xef, 0x41, 0x27, 0x2d, 0x7a, 0x29, + 0xa4, 0x4a, 0xc1, 0xe1, 0x09, 0xd3, 0x12, 0xbf, 0x95, 0xa0, 0xae, 0xd3, 0x69, 0x4a, 0xea, 0x7f, + 0x2c, 0xda, 0x75, 0x68, 0xb0, 0x2b, 0xc6, 0xc9, 0xf8, 0x48, 0x97, 0x6e, 0xfb, 0xdf, 0x96, 0xee, + 0x2f, 0x25, 0x68, 0xa4, 0x21, 0x5d, 0x3b, 0x7a, 0xde, 0x82, 0x46, 0xa4, 0x82, 0x23, 0xaa, 0x82, + 0x9b, 0x07, 0x1d, 0x8d, 0xdb, 0x04, 0x3d, 0x21, 0xa4, 0x5a, 0x18, 0x35, 0x0a, 0x3f, 0x86, 0x16, + 0x89, 0xfa, 0xaf, 0x89, 0xfa, 0xb7, 0x56, 0xa1, 0x1e, 0x27, 0x01, 0xa7, 0x98, 0x7e, 0xd9, 0x79, + 0xf6, 0x5d, 0xa8, 0xbf, 0x74, 0xdc, 0x11, 0xa2, 0x11, 0x92, 0x6e, 0xa4, 0x89, 0x95, 0x93, 0x75, + 0x4c, 0xc6, 0x61, 0x7c, 0x25, 0xa1, 0x54, 0xed, 0x6f, 0x71, 0xe4, 0xa8, 0x34, 0xe9, 0xfc, 0xde, + 0xc1, 0x6e, 0x30, 0x81, 0x98, 0xf4, 0x4e, 0x4d, 0x2a, 0xeb, 0x16, 0xd4, 0xc7, 0xca, 0xbe, 0x6e, + 0x18, 0x83, 0x5f, 0x7b, 0xb5, 0x9f, 0xc0, 0xb6, 0x9a, 0xd8, 0x0b, 0xe7, 0xf2, 0xd4, 0x4c, 0x53, + 0x21, 0xcb, 0x61, 0x6c, 0xef, 0xc2, 0xce, 0x94, 0x09, 0xdd, 0x1e, 0x36, 0xb4, 0x9f, 0x9d, 0x13, + 0xac, 0x3f, 0x63, 0x14, 0x33, 0x28, 0xa2, 0x47, 0xfd, 0x71, 0x24, 0x6d, 0x57, 0xed, 0x6f, 0x60, + 0x59, 0xca, 0x08, 0x02, 0x04, 0x36, 0xed, 0x52, 0xb9, 0x9f, 0xe5, 0xb1, 0x6d, 0xe0, 0x54, 0x4d, + 0x51, 0x4c, 0x4c, 0x2e, 0x4b, 0x93, 0x7f, 0x94, 0xa0, 0xf5, 0x8a, 0xf0, 0x8b, 0x30, 0x3e, 0x13, + 0xa4, 0xb1, 0x42, 0xcf, 0x61, 0x6d, 0xc6, 0x97, 0xfd, 0xc1, 0x15, 0xc7, 0xac, 0x4a, 0x76, 0x45, + 0xf2, 0x71, 0xe7, 0xc8, 0x51, 0x9d, 0x56, 0x91, 0x7b, 0x68, 0xf7, 0xf5, 0x65, 0x1f, 0x4b, 0x27, + 0x8c, 0x55, 0x72, 0xa5, 0x18, 0x6e, 0x79, 0x71, 0x18, 0x45, 0xc4, 0x53, 0xbe, 0x84, 0xb1, 0x13, + 0x63, 0xac, 0x66, 0xa4, 0x70, 0x27, 0xd2, 0xc6, 0xea, 0xc6, 0xd8, 0x49, 0x6a, 0x6c, 0x25, 0x23, + 0x66, 0x8c, 0x35, 0x24, 0xf0, 0x31, 0xac, 0x1c, 0x46, 0xc9, 0x29, 0x73, 0x86, 0x44, 0xb4, 0x3f, + 0x0f, 0xb9, 0xe3, 0xf7, 0x13, 0xb1, 0x54, 0x64, 0x59, 0x9b, 0xd0, 0x8a, 0x48, 0x8c, 0x75, 0xa2, + 0x77, 0xcb, 0x98, 0xf7, 0xaa, 0x75, 0x03, 0x36, 0xe4, 0xb2, 0x4f, 0x83, 0xfe, 0x19, 0x89, 0x03, + 0xe2, 0x8f, 0x43, 0x8f, 0xe8, 0x38, 0x76, 0x61, 0x3d, 0x3d, 0x14, 0x0d, 0x28, 0x8f, 0x64, 0x3c, + 0xf6, 0x09, 0x74, 0x4e, 0x46, 0x78, 0x47, 0xe0, 0x3e, 0x0d, 0x86, 0x4f, 0x1d, 0xee, 0x88, 0x02, + 0x45, 0xfb, 0x34, 0xf4, 0x98, 0x76, 0x88, 0xda, 0x5c, 0x89, 0x10, 0xaf, 0x6f, 0x8e, 0x14, 0x69, + 0x38, 0xe4, 0x27, 0x47, 0xb2, 0xa6, 0xa5, 0x43, 0x9b, 0xcb, 0x20, 0x14, 0xf1, 0x36, 0x7e, 0x4e, + 0x53, 0xb0, 0xea, 0x73, 0xba, 0x6a, 0x8a, 0xd4, 0x04, 0xba, 0x0f, 0xab, 0x3c, 0x45, 0xd1, 0xc7, + 0x42, 0x72, 0x74, 0xad, 0x6e, 0x69, 0xc9, 0x02, 0x46, 0xe4, 0x40, 0x4d, 0x01, 0x6d, 0x56, 0x79, + 0x7d, 0x05, 0xf0, 0x52, 0x36, 0x8c, 0x94, 0xc1, 0xb1, 0x90, 0xa5, 0x0d, 0xe9, 0x1f, 0x3b, 0x97, + 0x29, 0x67, 0x62, 0x0b, 0x23, 0xfd, 0xde, 0xa1, 0xbe, 0xab, 0xef, 0x14, 0x55, 0xa1, 0xe2, 0xd3, + 0x31, 0x8e, 0x62, 0xc5, 0xcd, 0x5f, 0x25, 0x68, 0x2a, 0x83, 0x2a, 0x12, 0x3c, 0x76, 0xb1, 0x67, + 0x8c, 0xc5, 0x3d, 0xe3, 0x20, 0xff, 0x1d, 0xca, 0x40, 0xc0, 0xcf, 0x15, 0xbb, 0x70, 0xa2, 0x0c, + 0xc8, 0x99, 0x62, 0x77, 0xa1, 0xa5, 0x52, 0xa6, 0x05, 0xab, 0xf3, 0x04, 0xef, 0x8b, 0x49, 0x87, + 0x48, 0xe4, 0x60, 0x69, 0x1e, 0xdc, 0xcc, 0x49, 0x48, 0x8c, 0xfb, 0xf2, 0xf7, 0x59, 0xc0, 0xe3, + 0xab, 0xde, 0x7d, 0x80, 0xc9, 0x4a, 0x34, 0xcc, 0x19, 0xb9, 0xd2, 0xe5, 0x8f, 0x91, 0x9c, 0x3b, + 0x7e, 0xa2, 0x89, 0x78, 0x5c, 0x7e, 0x54, 0xb2, 0xbf, 0x82, 0xd5, 0xcf, 0xfd, 0x33, 0x1a, 0x66, + 0x54, 0x50, 0x6a, 0xec, 0xfc, 0x10, 0xc6, 0x3a, 0x5e, 0xb1, 0xa4, 0x01, 0x2e, 0x15, 0x7b, 0xd8, + 0x9d, 0x61, 0x34, 0xb9, 0x8c, 0x29, 0x7b, 0x8a, 0xb8, 0x3f, 0x2b, 0x00, 0x13, 0x63, 0xd6, 0x63, + 0xe8, 0xd1, 0xb0, 0x8f, 0x75, 0x77, 0x4e, 0x5d, 0xa2, 0xfa, 0xa4, 0x1f, 0x13, 0x37, 0x89, 0x19, + 0x3d, 0x27, 0x7a, 0x6e, 0x6d, 0xeb, 0x58, 0x8a, 0x18, 0x3e, 0x80, 0xad, 0x89, 0xae, 0x97, 0x51, + 0x2b, 0x2f, 0x54, 0x7b, 0x08, 0x1b, 0xa8, 0x86, 0x03, 0x27, 0xc9, 0x29, 0x55, 0x16, 0x2a, 0x7d, + 0x0c, 0xbb, 0x19, 0x9c, 0xa2, 0x9c, 0x33, 0xaa, 0xd5, 0x85, 0xaa, 0x1f, 0xc2, 0x36, 0xaa, 0x5e, + 0x38, 0x94, 0x17, 0xf5, 0x96, 0xff, 0x01, 0xce, 0x31, 0x89, 0x87, 0x39, 0x9c, 0xb5, 0x85, 0x4a, + 0xef, 0xc1, 0x3a, 0x2a, 0x15, 0xfc, 0xd4, 0xaf, 0x53, 0x61, 0xc4, 0xe5, 0x38, 0x7a, 0x32, 0x2a, + 0x2b, 0x8b, 0x54, 0xec, 0x23, 0x68, 0x3d, 0x4f, 0x86, 0x84, 0xfb, 0x83, 0xb4, 0xfa, 0xff, 0x63, + 0x3f, 0xfd, 0x54, 0x86, 0xe6, 0xe1, 0x30, 0x0e, 0x93, 0x28, 0x37, 0x19, 0x54, 0x49, 0x4f, 0x4d, + 0x06, 0x25, 0x73, 0x0f, 0x5a, 0xea, 0x23, 0xa8, 0xc5, 0x54, 0xaf, 0x59, 0xd3, 0x95, 0x2f, 0x6e, + 0x3b, 0x03, 0x11, 0x82, 0x16, 0xcc, 0x77, 0x5b, 0xa6, 0x1a, 0x3f, 0x81, 0xf6, 0x48, 0xc5, 0xa5, + 0x25, 0x55, 0x66, 0xef, 0x18, 0xcf, 0x13, 0x80, 0xfb, 0xd9, 0xf8, 0x55, 0x4f, 0x3d, 0x87, 0xf5, + 0xa9, 0xcd, 0x7c, 0x6b, 0xd9, 0xd9, 0xd6, 0x6a, 0x1e, 0x6c, 0x68, 0xb3, 0x59, 0x2d, 0xd9, 0x6f, + 0x97, 0xea, 0x6b, 0x9e, 0x5e, 0x81, 0xad, 0x77, 0xa1, 0x1d, 0xa8, 0x0f, 0x56, 0xca, 0x48, 0x25, + 0x63, 0x20, 0xf7, 0x31, 0x43, 0x56, 0x5c, 0x89, 0x73, 0x26, 0x2b, 0x59, 0x8e, 0x73, 0x9f, 0x46, + 0x35, 0x26, 0xf5, 0x75, 0x6f, 0xd6, 0x43, 0xe3, 0xe0, 0xd7, 0x1a, 0x54, 0x9e, 0x1c, 0x7d, 0x69, + 0xbd, 0x86, 0xd5, 0xc2, 0xf3, 0xc8, 0x32, 0xd3, 0x66, 0xf6, 0x1b, 0xae, 0xf7, 0xe6, 0xbc, 0x63, + 0x7d, 0x0f, 0x58, 0x12, 0x36, 0x0b, 0x97, 0x84, 0xd4, 0xe6, 0xec, 0xfb, 0x47, 0x6a, 0x73, 0xde, + 0xdd, 0x62, 0xc9, 0xfa, 0x08, 0x6a, 0xea, 0x31, 0x65, 0x6d, 0x6a, 0xd9, 0xdc, 0xab, 0xac, 0xb7, + 0x55, 0xd8, 0x4d, 0x15, 0x5f, 0x40, 0x3b, 0xf7, 0x4c, 0xb5, 0x6e, 0xe4, 0x7c, 0xe5, 0xdf, 0x62, + 0xbd, 0x37, 0x66, 0x1f, 0xa6, 0xd6, 0x0e, 0x01, 0x26, 0x8f, 0x1a, 0xab, 0xab, 0xa5, 0xa7, 0xde, + 0x74, 0xbd, 0xdd, 0x19, 0x27, 0xa9, 0x91, 0x53, 0x58, 0x2b, 0xbe, 0x5a, 0xac, 0x02, 0xab, 0xc5, + 0x37, 0x46, 0xef, 0xd6, 0xdc, 0xf3, 0xac, 0xd9, 0xe2, 0xdb, 0x25, 0x35, 0x3b, 0xe7, 0x25, 0x94, + 0x9a, 0x9d, 0xfb, 0xe8, 0x59, 0xb2, 0xbe, 0x86, 0x4e, 0xfe, 0xd9, 0x61, 0x19, 0x92, 0x66, 0xbe, + 0x86, 0x7a, 0x37, 0xe7, 0x9c, 0xa6, 0x06, 0xdf, 0x87, 0x65, 0xf5, 0xc0, 0x30, 0x15, 0x9f, 0x7d, + 0x93, 0xf4, 0x36, 0xf3, 0x9b, 0xa9, 0xd6, 0x03, 0xa8, 0xa9, 0xeb, 0x65, 0x5a, 0x00, 0xb9, 0xdb, + 0x66, 0xaf, 0x95, 0xdd, 0xb5, 0x97, 0x1e, 0x94, 0x8c, 0x1f, 0x96, 0xf3, 0xc3, 0x66, 0xf9, 0xc9, + 0x24, 0x67, 0x50, 0x93, 0x7f, 0xa4, 0x3c, 0xfc, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x0a, 0x94, 0xe5, + 0x0c, 0x55, 0x11, 0x00, 0x00, } diff --git a/api/grpc/types/api.proto b/api/grpc/types/api.proto index c47b9b7..4a8d516 100644 --- a/api/grpc/types/api.proto +++ b/api/grpc/types/api.proto @@ -132,6 +132,7 @@ message Container { string status = 4; // Container status ("running", "paused", etc.) repeated string labels = 5; repeated uint32 pids = 6; + string runtime = 7; // runtime used to execute the container } // Machine is information about machine on which containerd is run diff --git a/containerd-shim/main.go b/containerd-shim/main.go index ece4cf7..544fba2 100644 --- a/containerd-shim/main.go +++ b/containerd-shim/main.go @@ -12,15 +12,18 @@ import ( "github.com/docker/docker/pkg/term" ) +var runtimeLog string + func setupLogger() { f, err := os.OpenFile("/tmp/shim.log", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0755) if err != nil { panic(err) } logrus.SetOutput(f) + runtimeLog = "/tmp/runtime.log" } -// containerd-shim is a small shim that sits in front of a runc implementation +// containerd-shim is a small shim that sits in front of a runtime implementation // that allows it to be repartented to init and handle reattach from the caller. // // the cwd of the shim should be the bundle for the container. Arg1 should be the path @@ -28,31 +31,43 @@ func setupLogger() { func main() { flag.Parse() // start handling signals as soon as possible so that things are properly reaped - // or if runc exits before we hit the handler + // or if runtime exits before we hit the handler signals := make(chan os.Signal, 2048) signal.Notify(signals) // set the shim as the subreaper for all orphaned processes created by the container if err := osutils.SetSubreaper(1); err != nil { - logrus.WithField("error", err).Fatal("shim: set as subreaper") + logrus.WithField("error", err).Error("shim: set as subreaper") + return } // open the exit pipe f, err := os.OpenFile("exit", syscall.O_WRONLY, 0) if err != nil { - logrus.WithField("error", err).Fatal("shim: open exit pipe") + logrus.WithField("error", err).Error("shim: open exit pipe") + return } defer f.Close() control, err := os.OpenFile("control", syscall.O_RDWR, 0) if err != nil { - logrus.WithField("error", err).Fatal("shim: open control pipe") + logrus.WithField("error", err).Error("shim: open control pipe") + return } defer control.Close() - p, err := newProcess(flag.Arg(0), flag.Arg(1)) + p, err := newProcess(flag.Arg(0), flag.Arg(1), flag.Arg(2)) if err != nil { - logrus.WithField("error", err).Fatal("shim: create new process") + logrus.WithField("error", err).Error("shim: create new process") + return } + defer func() { + if err := p.Close(); err != nil { + logrus.WithField("error", err).Error("shim: close stdio") + } + if err := p.delete(); err != nil { + logrus.WithField("error", err).Error("shim: delete runtime state") + } + }() if err := p.start(); err != nil { - p.delete() - logrus.WithField("error", err).Fatal("shim: start process") + logrus.WithField("error", err).Error("shim: start process") + return } go func() { for { @@ -87,13 +102,13 @@ func main() { logrus.WithField("error", err).Error("shim: reaping child processes") } for _, e := range exits { - // check to see if runc is one of the processes that has exited + // check to see if runtime is one of the processes that has exited if e.Pid == p.pid() { exitShim = true logrus.WithFields(logrus.Fields{ "pid": e.Pid, "status": e.Status, - }).Info("shim: runc exited") + }).Info("shim: runtime exited") if err := writeInt("exitStatus", e.Status); err != nil { logrus.WithFields(logrus.Fields{ "error": err, @@ -103,14 +118,8 @@ func main() { } } } - // runc has exited so the shim can also exit + // runtime has exited so the shim can also exit if exitShim { - if err := p.Close(); err != nil { - logrus.WithField("error", err).Error("shim: close stdio") - } - if err := p.delete(); err != nil { - logrus.WithField("error", err).Error("shim: delete runc state") - } return } } diff --git a/containerd-shim/process.go b/containerd-shim/process.go index c7a3e37..b97b900 100644 --- a/containerd-shim/process.go +++ b/containerd-shim/process.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "io" "io/ioutil" "os" @@ -25,12 +26,14 @@ type process struct { console libcontainer.Console consolePath string state *runtime.ProcessState + runtime string } -func newProcess(id, bundle string) (*process, error) { +func newProcess(id, bundle, runtimeName string) (*process, error) { p := &process{ - id: id, - bundle: bundle, + id: id, + bundle: bundle, + runtime: runtimeName, } s, err := loadProcess() if err != nil { @@ -81,7 +84,7 @@ func (p *process) start() error { if err != nil { return err } - args := []string{} + args := []string{"--log", runtimeLog} if p.state.Exec { args = append(args, "exec", "--process", filepath.Join(cwd, "process.json"), @@ -114,7 +117,7 @@ func (p *process) start() error { "--pid-file", filepath.Join(cwd, "pid"), p.id, ) - cmd := exec.Command("runc", args...) + cmd := exec.Command(p.runtime, args...) cmd.Dir = p.bundle cmd.Stdin = p.stdio.stdin cmd.Stdout = p.stdio.stdout @@ -146,7 +149,10 @@ func (p *process) pid() int { func (p *process) delete() error { if !p.state.Exec { - return exec.Command("runc", "delete", p.id).Run() + out, err := exec.Command(p.runtime, "delete", p.id).CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %v", out, err) + } } return nil } diff --git a/containerd/main.go b/containerd/main.go index c9ef921..e713b3f 100644 --- a/containerd/main.go +++ b/containerd/main.go @@ -41,6 +41,11 @@ var daemonFlags = []cli.Flag{ Value: defaultGRPCEndpoint, Usage: "Address on which GRPC API will listen", }, + cli.StringFlag{ + Name: "runtime,r", + Value: "runc", + Usage: "name of the OCI compliant runtime to use when executing containers", + }, } func main() { @@ -58,6 +63,7 @@ func main() { context.String("state-dir"), 10, context.Bool("oom-notify"), // TODO Windows: Remove oom-notify + context.String("runtime"), ); err != nil { logrus.Fatal(err) } @@ -67,11 +73,11 @@ func main() { } } -func daemon(address, stateDir string, concurrency int, oom bool) error { +func daemon(address, stateDir string, concurrency int, oom bool, runtimeName string) error { // setup a standard reaper so that we don't leave any zombies if we are still alive // this is just good practice because we are spawning new processes go reapProcesses() - sv, err := supervisor.New(stateDir, oom) + sv, err := supervisor.New(stateDir, oom, runtimeName) if err != nil { return err } diff --git a/ctr/container.go b/ctr/container.go index d489e3b..6753c97 100644 --- a/ctr/container.go +++ b/ctr/container.go @@ -92,6 +92,7 @@ func listContainers(context *cli.Context) { } w := tabwriter.NewWriter(os.Stdout, 20, 1, 3, ' ', 0) fmt.Fprint(w, "ID\tPATH\tSTATUS\tPROCESSES\n") + sortContainers(resp.Containers) for _, c := range resp.Containers { procs := []string{} for _, p := range c.Processes { diff --git a/ctr/sort.go b/ctr/sort.go new file mode 100644 index 0000000..fbd090b --- /dev/null +++ b/ctr/sort.go @@ -0,0 +1,27 @@ +package main + +import ( + "sort" + + "github.com/docker/containerd/api/grpc/types" +) + +func sortContainers(c []*types.Container) { + sort.Sort(&containerSorter{c}) +} + +type containerSorter struct { + c []*types.Container +} + +func (s *containerSorter) Len() int { + return len(s.c) +} + +func (s *containerSorter) Swap(i, j int) { + s.c[i], s.c[j] = s.c[j], s.c[i] +} + +func (s *containerSorter) Less(i, j int) bool { + return s.c[i].Id < s.c[j].Id +} diff --git a/runtime/container.go b/runtime/container.go index d07b69b..93a0f90 100644 --- a/runtime/container.go +++ b/runtime/container.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "path/filepath" + "syscall" "github.com/Sirupsen/logrus" ) @@ -42,6 +43,8 @@ type Container interface { Pids() ([]int, error) // Stats returns realtime container stats and resource information Stats() (*Stat, error) + // Name or path of the OCI compliant runtime used to execute the container + Runtime() string // OOM signals the channel if the container received an OOM notification // OOM() (<-chan struct{}, error) } @@ -68,13 +71,14 @@ func NewStdio(stdin, stdout, stderr string) Stdio { } // New returns a new container -func New(root, id, bundle string, labels []string) (Container, error) { +func New(root, id, bundle, runtimeName string, labels []string) (Container, error) { c := &container{ root: root, id: id, bundle: bundle, labels: labels, processes: make(map[string]*process), + runtime: runtimeName, } if err := os.Mkdir(filepath.Join(root, id), 0755); err != nil { return nil, err @@ -85,8 +89,9 @@ func New(root, id, bundle string, labels []string) (Container, error) { } defer f.Close() if err := json.NewEncoder(f).Encode(state{ - Bundle: bundle, - Labels: labels, + Bundle: bundle, + Labels: labels, + Runtime: runtimeName, }); err != nil { return nil, err } @@ -108,6 +113,7 @@ func Load(root, id string) (Container, error) { id: id, bundle: s.Bundle, labels: s.Labels, + runtime: s.Runtime, processes: make(map[string]*process), } dirs, err := ioutil.ReadDir(filepath.Join(root, id)) @@ -151,6 +157,7 @@ type container struct { root string id string bundle string + runtime string processes map[string]*process stdio Stdio labels []string @@ -182,6 +189,14 @@ func (c *container) readSpec() (*PlatformSpec, error) { } func (c *container) State() State { + proc := c.processes["init"] + if proc == nil || proc.pid == 0 { + return Stopped + } + err := syscall.Kill(proc.pid, 0) + if err != nil && err == syscall.ESRCH { + return Stopped + } return Running } diff --git a/runtime/container_linux.go b/runtime/container_linux.go index 1734977..6f6a501 100644 --- a/runtime/container_linux.go +++ b/runtime/container_linux.go @@ -32,6 +32,10 @@ func getRootIDs(s *PlatformSpec) (int, int, error) { return uid, gid, nil } +func (c *container) Runtime() string { + return c.runtime +} + func (c *container) Pause() error { return exec.Command("runc", "pause", c.id).Run() } @@ -115,7 +119,7 @@ func (c *container) Start(checkpoint string, s Stdio) (Process, error) { return nil, err } cmd := exec.Command("containerd-shim", - c.id, c.bundle, + c.id, c.bundle, c.runtime, ) cmd.Dir = processRoot cmd.SysProcAttr = &syscall.SysProcAttr{ @@ -141,8 +145,8 @@ func (c *container) Start(checkpoint string, s Stdio) (Process, error) { if err := cmd.Start(); err != nil { return nil, err } - if _, err := p.getPid(); err != nil { - return p, nil + if err := waitForStart(p, cmd); err != nil { + return nil, err } c.processes[InitProcessID] = p return p, nil @@ -154,7 +158,7 @@ func (c *container) Exec(pid string, spec ProcessSpec, s Stdio) (Process, error) return nil, err } cmd := exec.Command("containerd-shim", - c.id, c.bundle, + c.id, c.bundle, c.runtime, ) cmd.Dir = processRoot cmd.SysProcAttr = &syscall.SysProcAttr{ @@ -175,8 +179,8 @@ func (c *container) Exec(pid string, spec ProcessSpec, s Stdio) (Process, error) if err := cmd.Start(); err != nil { return nil, err } - if _, err := p.getPid(); err != nil { - return p, nil + if err := waitForStart(p, cmd); err != nil { + return nil, err } c.processes[pid] = p return p, nil @@ -222,3 +226,34 @@ func (c *container) Stats() (*Stat, error) { Data: stats, }, nil } + +func waitForStart(p *process, cmd *exec.Cmd) error { + for i := 0; i < 50; i++ { + if _, err := p.getPidFromFile(); err != nil { + if os.IsNotExist(err) { + alive, err := isAlive(cmd) + if err != nil { + return err + } + if !alive { + return ErrContainerNotStarted + } + time.Sleep(100 * time.Millisecond) + continue + } + return err + } + return nil + } + return errNoPidFile +} + +func isAlive(cmd *exec.Cmd) (bool, error) { + if err := syscall.Kill(cmd.Process.Pid, 0); err != nil { + if err == syscall.ESRCH { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/runtime/container_windows.go b/runtime/container_windows.go index 7d04653..a7348af 100644 --- a/runtime/container_windows.go +++ b/runtime/container_windows.go @@ -6,6 +6,10 @@ func getRootIDs(s *PlatformSpec) (int, int, error) { return 0, 0, nil } +func (c *container) Runtime() string { + return "windows" +} + func (c *container) Pause() error { return errors.New("Pause not supported on Windows") } diff --git a/runtime/process.go b/runtime/process.go index b1c08d5..c3b2674 100644 --- a/runtime/process.go +++ b/runtime/process.go @@ -8,7 +8,6 @@ import ( "os" "path/filepath" "strconv" - "time" ) type Process interface { @@ -95,7 +94,7 @@ func loadProcess(root, id string, c *container, s *ProcessState) (*process, erro Stderr: s.Stderr, }, } - if _, err := p.getPid(); err != nil { + if _, err := p.getPidFromFile(); err != nil { return nil, err } if _, err := p.ExitStatus(); err != nil { @@ -177,22 +176,15 @@ func (p *process) Close() error { return p.exitPipe.Close() } -func (p *process) getPid() (int, error) { - for i := 0; i < 20; i++ { - data, err := ioutil.ReadFile(filepath.Join(p.root, "pid")) - if err != nil { - if os.IsNotExist(err) { - time.Sleep(100 * time.Millisecond) - continue - } - return -1, err - } - i, err := strconv.Atoi(string(data)) - if err != nil { - return -1, err - } - p.pid = i - return i, nil +func (p *process) getPidFromFile() (int, error) { + data, err := ioutil.ReadFile(filepath.Join(p.root, "pid")) + if err != nil { + return -1, err } - return -1, fmt.Errorf("containerd: cannot read pid file") + i, err := strconv.Atoi(string(data)) + if err != nil { + return -1, err + } + p.pid = i + return i, nil } diff --git a/runtime/runtime.go b/runtime/runtime.go index 26d9465..5727593 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -14,7 +14,9 @@ var ( ErrTerminalsNotSupported = errors.New("containerd: terminals are not supported for runtime") ErrProcessNotExited = errors.New("containerd: process has not exited") ErrProcessExited = errors.New("containerd: process has exited") + ErrContainerNotStarted = errors.New("containerd: container not started") + errNoPidFile = errors.New("containerd: no process pid file found") errNotImplemented = errors.New("containerd: not implemented") ) @@ -30,15 +32,17 @@ type State string const ( Paused = State("paused") + Stopped = State("stopped") Running = State("running") ) type state struct { - Bundle string `json:"bundle"` - Labels []string `json:"labels"` - Stdin string `json:"stdin"` - Stdout string `json:"stdout"` - Stderr string `json:"stderr"` + Bundle string `json:"bundle"` + Labels []string `json:"labels"` + Stdin string `json:"stdin"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + Runtime string `json:"runtime"` } type ProcessState struct { diff --git a/runtime/spec_linux.go b/runtime/spec_linux.go index d243cf4..147affb 100644 --- a/runtime/spec_linux.go +++ b/runtime/spec_linux.go @@ -2,5 +2,7 @@ package runtime import "github.com/opencontainers/specs" -type PlatformSpec specs.LinuxSpec -type ProcessSpec specs.Process +type ( + PlatformSpec specs.LinuxSpec + ProcessSpec specs.Process +) diff --git a/supervisor/create.go b/supervisor/create.go index 68ea707..16ad36e 100644 --- a/supervisor/create.go +++ b/supervisor/create.go @@ -20,7 +20,7 @@ type StartTask struct { func (s *Supervisor) start(t *StartTask) error { start := time.Now() - container, err := runtime.New(s.stateDir, t.ID, t.BundlePath, t.Labels) + container, err := runtime.New(s.stateDir, t.ID, t.BundlePath, s.runtime, t.Labels) if err != nil { return err } diff --git a/supervisor/exit.go b/supervisor/exit.go index 5e179cf..ef0b314 100644 --- a/supervisor/exit.go +++ b/supervisor/exit.go @@ -17,9 +17,19 @@ func (s *Supervisor) exit(t *ExitTask) error { proc := t.Process status, err := proc.ExitStatus() if err != nil { - logrus.WithField("error", err).Error("containerd: get exit status") + logrus.WithFields(logrus.Fields{ + "error": err, + "pid": proc.ID(), + "id": proc.Container().ID(), + "systemPid": proc.SystemPid(), + }).Error("containerd: get exit status") } - logrus.WithFields(logrus.Fields{"pid": proc.ID(), "status": status}).Debug("containerd: process exited") + logrus.WithFields(logrus.Fields{ + "pid": proc.ID(), + "status": status, + "id": proc.Container().ID(), + "systemPid": proc.SystemPid(), + }).Debug("containerd: process exited") // if the process is the the init process of the container then // fire a separate event for this process diff --git a/supervisor/sort.go b/supervisor/sort.go new file mode 100644 index 0000000..2153b7e --- /dev/null +++ b/supervisor/sort.go @@ -0,0 +1,27 @@ +package supervisor + +import ( + "sort" + + "github.com/docker/containerd/runtime" +) + +func sortProcesses(p []runtime.Process) { + sort.Sort(&processSorter{p}) +} + +type processSorter struct { + processes []runtime.Process +} + +func (s *processSorter) Len() int { + return len(s.processes) +} + +func (s *processSorter) Swap(i, j int) { + s.processes[i], s.processes[j] = s.processes[j], s.processes[i] +} + +func (s *processSorter) Less(i, j int) bool { + return s.processes[j].ID() == "init" +} diff --git a/supervisor/supervisor.go b/supervisor/supervisor.go index 4ff9b53..7b11080 100644 --- a/supervisor/supervisor.go +++ b/supervisor/supervisor.go @@ -6,7 +6,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sort" "sync" "time" @@ -20,7 +19,7 @@ const ( ) // New returns an initialized Process supervisor. -func New(stateDir string, oom bool) (*Supervisor, error) { +func New(stateDir string, oom bool, runtimeName string) (*Supervisor, error) { startTasks := make(chan *startTask, 10) if err := os.MkdirAll(stateDir, 0755); err != nil { return nil, err @@ -41,6 +40,7 @@ func New(stateDir string, oom bool) (*Supervisor, error) { subscribers: make(map[chan Event]struct{}), tasks: make(chan Task, defaultBufferSize), monitor: monitor, + runtime: runtimeName, } if err := setupEventLog(s); err != nil { return nil, err @@ -116,7 +116,9 @@ func readEventLog(s *Supervisor) error { type Supervisor struct { // stateDir is the directory on the system to store container runtime state information. - stateDir string + stateDir string + // name of the OCI compatible runtime used to execute containers + runtime string containers map[string]*containerInfo startTasks chan *startTask // we need a lock around the subscribers map only because additions and deletions from @@ -207,7 +209,12 @@ func (s *Supervisor) notifySubscribers(e Event) { // therefore it is save to do operations in the handlers that modify state of the system or // state of the Supervisor func (s *Supervisor) Start() error { - logrus.WithField("stateDir", s.stateDir).Debug("containerd: supervisor running") + logrus.WithFields(logrus.Fields{ + "stateDir": s.stateDir, + "runtime": s.runtime, + "memory": s.machine.Memory, + "cpus": s.machine.Cpus, + }).Debug("containerd: supervisor running") go func() { for i := range s.tasks { s.handleTask(i) @@ -277,7 +284,7 @@ func (s *Supervisor) restore() error { if len(exitedProcesses) > 0 { // sort processes so that init is fired last because that is how the kernel sends the // exit events - sort.Sort(&processSorter{exitedProcesses}) + sortProcesses(exitedProcesses) for _, p := range exitedProcesses { e := &ExitTask{ Process: p, @@ -288,19 +295,3 @@ func (s *Supervisor) restore() error { } return nil } - -type processSorter struct { - processes []runtime.Process -} - -func (s *processSorter) Len() int { - return len(s.processes) -} - -func (s *processSorter) Swap(i, j int) { - s.processes[i], s.processes[j] = s.processes[j], s.processes[i] -} - -func (s *processSorter) Less(i, j int) bool { - return s.processes[j].ID() == "init" -} diff --git a/supervisor/worker.go b/supervisor/worker.go index d08fdb4..b6e4dbd 100644 --- a/supervisor/worker.go +++ b/supervisor/worker.go @@ -40,11 +40,15 @@ func (w *worker) Start() { started := time.Now() process, err := t.Container.Start(t.Checkpoint, runtime.NewStdio(t.Stdin, t.Stdout, t.Stderr)) if err != nil { + logrus.WithFields(logrus.Fields{ + "error": err, + "id": t.Container.ID(), + }).Error("containerd: start container") + t.Err <- err evt := &DeleteTask{ ID: t.Container.ID(), } w.s.SendTask(evt) - t.Err <- err continue } /*