diff --git a/api/grpc/server/server.go b/api/grpc/server/server.go index d80e153..e3d86a2 100644 --- a/api/grpc/server/server.go +++ b/api/grpc/server/server.go @@ -8,10 +8,11 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" - "github.com/Sirupsen/logrus" "github.com/docker/containerd/api/grpc/types" "github.com/docker/containerd/runtime" "github.com/docker/containerd/supervisor" + "github.com/opencontainers/runc/libcontainer" + "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/specs" "golang.org/x/net/context" ) @@ -297,38 +298,87 @@ func (s *apiServer) Events(r *types.EventsRequest, stream types.API_EventsServer return nil } -func (s *apiServer) GetStats(r *types.StatsRequest, stream types.API_GetStatsServer) error { +func (s *apiServer) Stats(ctx context.Context, r *types.StatsRequest) (*types.StatsResponse, error) { e := supervisor.NewEvent(supervisor.StatsEventType) e.ID = r.Id + e.Stat = make(chan *runtime.Stat, 1) s.sv.SendEvent(e) if err := <-e.Err; err != nil { - if err == supervisor.ErrContainerNotFound { - return grpc.Errorf(codes.NotFound, err.Error()) - } - return err + return nil, err } - defer func() { - ue := supervisor.NewEvent(supervisor.UnsubscribeStatsEventType) - ue.ID = e.ID - ue.Stats = e.Stats - s.sv.SendEvent(ue) - if err := <-ue.Err; err != nil { - logrus.Errorf("Error unsubscribing %s: %v", r.Id, err) - } - }() - for { - select { - case st := <-e.Stats: - pbSt, ok := st.(*types.Stats) - if !ok { - panic("invalid stats type from collector") - } - if err := stream.Send(pbSt); err != nil { - return err - } - case <-stream.Context().Done(): - return nil - } - } - return nil + stats := <-e.Stat + t := convertToPb(stats) + return t, nil +} + +func convertToPb(st *runtime.Stat) *types.StatsResponse { + pbSt := &types.StatsResponse{ + Timestamp: uint64(st.Timestamp.Unix()), + CgroupStats: &types.CgroupStats{}, + } + lcSt, ok := st.Data.(*libcontainer.Stats) + if !ok { + return pbSt + } + cpuSt := lcSt.CgroupStats.CpuStats + pbSt.CgroupStats.CpuStats = &types.CpuStats{ + CpuUsage: &types.CpuUsage{ + TotalUsage: cpuSt.CpuUsage.TotalUsage, + PercpuUsage: cpuSt.CpuUsage.PercpuUsage, + UsageInKernelmode: cpuSt.CpuUsage.UsageInKernelmode, + UsageInUsermode: cpuSt.CpuUsage.UsageInUsermode, + }, + ThrottlingData: &types.ThrottlingData{ + Periods: cpuSt.ThrottlingData.Periods, + ThrottledPeriods: cpuSt.ThrottlingData.ThrottledPeriods, + ThrottledTime: cpuSt.ThrottlingData.ThrottledTime, + }, + } + memSt := lcSt.CgroupStats.MemoryStats + pbSt.CgroupStats.MemoryStats = &types.MemoryStats{ + Cache: memSt.Cache, + Usage: &types.MemoryData{ + Usage: memSt.Usage.Usage, + MaxUsage: memSt.Usage.MaxUsage, + Failcnt: memSt.Usage.Failcnt, + }, + SwapUsage: &types.MemoryData{ + Usage: memSt.SwapUsage.Usage, + MaxUsage: memSt.SwapUsage.MaxUsage, + Failcnt: memSt.SwapUsage.Failcnt, + }, + } + blkSt := lcSt.CgroupStats.BlkioStats + pbSt.CgroupStats.BlkioStats = &types.BlkioStats{ + IoServiceBytesRecursive: convertBlkioEntryToPb(blkSt.IoServiceBytesRecursive), + IoServicedRecursive: convertBlkioEntryToPb(blkSt.IoServicedRecursive), + IoQueuedRecursive: convertBlkioEntryToPb(blkSt.IoQueuedRecursive), + IoServiceTimeRecursive: convertBlkioEntryToPb(blkSt.IoServiceTimeRecursive), + IoWaitTimeRecursive: convertBlkioEntryToPb(blkSt.IoWaitTimeRecursive), + IoMergedRecursive: convertBlkioEntryToPb(blkSt.IoMergedRecursive), + IoTimeRecursive: convertBlkioEntryToPb(blkSt.IoTimeRecursive), + SectorsRecursive: convertBlkioEntryToPb(blkSt.SectorsRecursive), + } + pbSt.CgroupStats.HugetlbStats = make(map[string]*types.HugetlbStats) + for k, st := range lcSt.CgroupStats.HugetlbStats { + pbSt.CgroupStats.HugetlbStats[k] = &types.HugetlbStats{ + Usage: st.Usage, + MaxUsage: st.MaxUsage, + Failcnt: st.Failcnt, + } + } + return pbSt +} + +func convertBlkioEntryToPb(b []cgroups.BlkioStatEntry) []*types.BlkioStatsEntry { + var pbEs []*types.BlkioStatsEntry + for _, e := range b { + pbEs = append(pbEs, &types.BlkioStatsEntry{ + Major: e.Major, + Minor: e.Minor, + Op: e.Op, + Value: e.Value, + }) + } + return pbEs } diff --git a/api/grpc/types/api.pb.go b/api/grpc/types/api.pb.go index 02fa50d..ff143bc 100644 --- a/api/grpc/types/api.pb.go +++ b/api/grpc/types/api.pb.go @@ -45,7 +45,7 @@ It has these top-level messages: BlkioStats HugetlbStats CgroupStats - Stats + StatsResponse StatsRequest */ package types @@ -677,25 +677,25 @@ func (m *CgroupStats) GetHugetlbStats() map[string]*HugetlbStats { return nil } -type Stats struct { +type StatsResponse struct { NetworkStats []*NetworkStats `protobuf:"bytes,1,rep,name=network_stats" json:"network_stats,omitempty"` CgroupStats *CgroupStats `protobuf:"bytes,2,opt,name=cgroup_stats" json:"cgroup_stats,omitempty"` Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp" json:"timestamp,omitempty"` } -func (m *Stats) Reset() { *m = Stats{} } -func (m *Stats) String() string { return proto.CompactTextString(m) } -func (*Stats) ProtoMessage() {} -func (*Stats) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} } +func (m *StatsResponse) Reset() { *m = StatsResponse{} } +func (m *StatsResponse) String() string { return proto.CompactTextString(m) } +func (*StatsResponse) ProtoMessage() {} +func (*StatsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} } -func (m *Stats) GetNetworkStats() []*NetworkStats { +func (m *StatsResponse) GetNetworkStats() []*NetworkStats { if m != nil { return m.NetworkStats } return nil } -func (m *Stats) GetCgroupStats() *CgroupStats { +func (m *StatsResponse) GetCgroupStats() *CgroupStats { if m != nil { return m.CgroupStats } @@ -748,7 +748,7 @@ func init() { proto.RegisterType((*BlkioStats)(nil), "types.BlkioStats") proto.RegisterType((*HugetlbStats)(nil), "types.HugetlbStats") proto.RegisterType((*CgroupStats)(nil), "types.CgroupStats") - proto.RegisterType((*Stats)(nil), "types.Stats") + proto.RegisterType((*StatsResponse)(nil), "types.StatsResponse") proto.RegisterType((*StatsRequest)(nil), "types.StatsRequest") } @@ -769,7 +769,7 @@ type APIClient interface { ListCheckpoint(ctx context.Context, in *ListCheckpointRequest, opts ...grpc.CallOption) (*ListCheckpointResponse, error) State(ctx context.Context, in *StateRequest, opts ...grpc.CallOption) (*StateResponse, error) Events(ctx context.Context, in *EventsRequest, opts ...grpc.CallOption) (API_EventsClient, error) - GetStats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (API_GetStatsClient, error) + Stats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (*StatsResponse, error) } type aPIClient struct { @@ -893,36 +893,13 @@ func (x *aPIEventsClient) Recv() (*Event, error) { return m, nil } -func (c *aPIClient) GetStats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (API_GetStatsClient, error) { - stream, err := grpc.NewClientStream(ctx, &_API_serviceDesc.Streams[1], c.cc, "/types.API/GetStats", opts...) +func (c *aPIClient) Stats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (*StatsResponse, error) { + out := new(StatsResponse) + err := grpc.Invoke(ctx, "/types.API/Stats", in, out, c.cc, opts...) if err != nil { return nil, err } - x := &aPIGetStatsClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type API_GetStatsClient interface { - Recv() (*Stats, error) - grpc.ClientStream -} - -type aPIGetStatsClient struct { - grpc.ClientStream -} - -func (x *aPIGetStatsClient) Recv() (*Stats, error) { - m := new(Stats) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil + return out, nil } // Server API for API service @@ -938,7 +915,7 @@ type APIServer interface { ListCheckpoint(context.Context, *ListCheckpointRequest) (*ListCheckpointResponse, error) State(context.Context, *StateRequest) (*StateResponse, error) Events(*EventsRequest, API_EventsServer) error - GetStats(*StatsRequest, API_GetStatsServer) error + Stats(context.Context, *StatsRequest) (*StatsResponse, error) } func RegisterAPIServer(s *grpc.Server, srv APIServer) { @@ -1074,25 +1051,16 @@ func (x *aPIEventsServer) Send(m *Event) error { return x.ServerStream.SendMsg(m) } -func _API_GetStats_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(StatsRequest) - if err := stream.RecvMsg(m); err != nil { - return err +func _API_Stats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) { + in := new(StatsRequest) + if err := dec(in); err != nil { + return nil, err } - return srv.(APIServer).GetStats(m, &aPIGetStatsServer{stream}) -} - -type API_GetStatsServer interface { - Send(*Stats) error - grpc.ServerStream -} - -type aPIGetStatsServer struct { - grpc.ServerStream -} - -func (x *aPIGetStatsServer) Send(m *Stats) error { - return x.ServerStream.SendMsg(m) + out, err := srv.(APIServer).Stats(ctx, in) + if err != nil { + return nil, err + } + return out, nil } var _API_serviceDesc = grpc.ServiceDesc{ @@ -1135,6 +1103,10 @@ var _API_serviceDesc = grpc.ServiceDesc{ MethodName: "State", Handler: _API_State_Handler, }, + { + MethodName: "Stats", + Handler: _API_Stats_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -1142,111 +1114,106 @@ var _API_serviceDesc = grpc.ServiceDesc{ Handler: _API_Events_Handler, ServerStreams: true, }, - { - StreamName: "GetStats", - Handler: _API_GetStats_Handler, - ServerStreams: true, - }, }, } var fileDescriptor0 = []byte{ - // 1544 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x6e, 0xdc, 0x54, - 0x18, 0xce, 0x8c, 0x3d, 0xdb, 0x3f, 0x4b, 0x12, 0x67, 0x9b, 0x4c, 0x29, 0x0d, 0x6e, 0xa1, 0x15, - 0xaa, 0xa2, 0x92, 0xb2, 0x94, 0x22, 0x01, 0x25, 0xad, 0x5a, 0x50, 0x0b, 0x51, 0x93, 0x20, 0x71, - 0xc3, 0xc8, 0x63, 0x1f, 0x66, 0x0e, 0xf1, 0x86, 0x7d, 0x9c, 0xe5, 0x15, 0xca, 0x25, 0xcf, 0x81, - 0xc4, 0x15, 0x0f, 0xc0, 0x13, 0xf0, 0x1c, 0x3c, 0x05, 0xff, 0x59, 0xec, 0xb1, 0x3d, 0x4b, 0xe9, - 0x05, 0x37, 0x23, 0x9d, 0x73, 0xfe, 0xe5, 0xfb, 0xbf, 0x7f, 0xf1, 0x39, 0x03, 0x2d, 0x2b, 0xa4, - 0xfb, 0x61, 0x14, 0xb0, 0xc0, 0xa8, 0xb1, 0xab, 0x90, 0xc4, 0xe6, 0x08, 0x36, 0x4f, 0x43, 0xc7, - 0x62, 0xe4, 0x28, 0x0a, 0x6c, 0x12, 0xc7, 0x2f, 0xc9, 0x2f, 0x09, 0x89, 0x99, 0x01, 0x50, 0xa5, - 0x4e, 0xbf, 0xb2, 0x57, 0xb9, 0xd3, 0x32, 0xda, 0xa0, 0x85, 0xb8, 0xa8, 0x8a, 0x05, 0x9e, 0xd8, - 0x6e, 0x10, 0x93, 0x63, 0xe6, 0x50, 0xbf, 0xaf, 0xe1, 0x5e, 0xd3, 0xe8, 0x42, 0xed, 0x82, 0x3a, - 0x6c, 0xd2, 0xd7, 0x71, 0xd9, 0x35, 0x7a, 0x50, 0x9f, 0x10, 0x3a, 0x9e, 0xb0, 0x7e, 0x8d, 0xaf, - 0xcd, 0x1d, 0xd8, 0x2a, 0xf9, 0x88, 0xc3, 0xc0, 0x8f, 0x89, 0xf9, 0x6b, 0x05, 0xb6, 0x0f, 0x23, - 0x82, 0x27, 0x87, 0x81, 0xcf, 0x2c, 0xea, 0x93, 0x68, 0x9e, 0x7f, 0x5c, 0x8c, 0x12, 0xdf, 0x71, - 0xc9, 0x91, 0x85, 0x3e, 0xa6, 0x30, 0x26, 0xc4, 0x3e, 0x0b, 0x03, 0xea, 0x33, 0x01, 0xa3, 0xc5, - 0x61, 0xc4, 0x02, 0x95, 0x2e, 0x96, 0x08, 0x03, 0x97, 0x41, 0x22, 0x61, 0xa4, 0x6b, 0x12, 0x45, - 0xfd, 0x7a, 0xba, 0x76, 0xad, 0x11, 0x71, 0xe3, 0x7e, 0x63, 0x4f, 0xbb, 0xd3, 0x32, 0x3f, 0x87, - 0x9d, 0x19, 0x30, 0x12, 0xa8, 0x71, 0x13, 0x5a, 0x76, 0xba, 0x29, 0x40, 0xb5, 0x0f, 0xd6, 0xf6, - 0x05, 0x81, 0xfb, 0x99, 0xb0, 0xf9, 0x00, 0xba, 0xc7, 0x74, 0xec, 0x5b, 0xee, 0x6b, 0x39, 0xe4, - 0x48, 0x84, 0xa4, 0x00, 0xde, 0x35, 0xd7, 0xa0, 0x97, 0x6a, 0x2a, 0x66, 0xfe, 0xa8, 0xc0, 0xfa, - 0x23, 0xc7, 0x59, 0x92, 0x94, 0x35, 0x68, 0x32, 0x12, 0x79, 0x94, 0x5b, 0xa9, 0x8a, 0x2c, 0xec, - 0x82, 0x9e, 0xc4, 0x88, 0x4f, 0x13, 0xf8, 0xda, 0x0a, 0xdf, 0x29, 0x6e, 0x19, 0x1d, 0xd0, 0xad, - 0x68, 0x1c, 0x23, 0x31, 0x9a, 0xc4, 0x42, 0xfc, 0x73, 0x64, 0x45, 0x2d, 0xec, 0x0b, 0x47, 0x51, - 0xa2, 0x50, 0x36, 0x8a, 0x74, 0x36, 0x4b, 0x74, 0xb6, 0x4a, 0x74, 0x02, 0x5f, 0x63, 0xf8, 0xba, - 0xf0, 0x85, 0x36, 0x12, 0x85, 0xb2, 0xcb, 0x17, 0x63, 0x15, 0x76, 0xd7, 0xd8, 0x86, 0x9e, 0xe5, - 0x38, 0x94, 0xd1, 0x00, 0x41, 0x3f, 0xa5, 0x4e, 0x8c, 0x50, 0x35, 0x0c, 0x7f, 0x13, 0x8c, 0x7c, - 0xac, 0x8a, 0x82, 0xe7, 0x59, 0x3a, 0xb2, 0x3c, 0xcf, 0xe3, 0xe1, 0xdd, 0x42, 0x21, 0x54, 0x45, - 0xec, 0xeb, 0x69, 0x6e, 0xb2, 0x03, 0x73, 0x00, 0xfd, 0x59, 0x6b, 0xca, 0xd3, 0x7d, 0xd8, 0x79, - 0x4c, 0x5c, 0xf2, 0x3a, 0x4f, 0x48, 0xa2, 0x6f, 0x79, 0x44, 0xe6, 0x90, 0x1b, 0x9c, 0x55, 0x52, - 0x06, 0x6f, 0xc2, 0xd6, 0x73, 0x1a, 0xb3, 0xa5, 0xe6, 0xcc, 0x1f, 0x00, 0xa6, 0x02, 0x99, 0xf1, - 0xcc, 0x15, 0xb9, 0xa4, 0x4c, 0x25, 0x16, 0x49, 0x64, 0x76, 0xa8, 0x7a, 0x6d, 0x03, 0xda, 0x89, - 0x4f, 0x2f, 0x8f, 0x03, 0xfb, 0x8c, 0xb0, 0x58, 0x94, 0xba, 0x68, 0xc0, 0x78, 0x42, 0x5c, 0x57, - 0x54, 0x7a, 0xd3, 0xfc, 0x12, 0xb6, 0xcb, 0xfe, 0x55, 0x21, 0xbf, 0x07, 0xed, 0x29, 0x5b, 0x31, - 0x7a, 0xd3, 0x16, 0xd1, 0xd5, 0x39, 0x66, 0xc8, 0xd6, 0x3c, 0xe0, 0x7b, 0xd0, 0xcb, 0x8a, 0x5e, - 0x08, 0xc9, 0x52, 0xb0, 0x58, 0x12, 0x2b, 0x89, 0xdf, 0x2b, 0xd0, 0x50, 0xe9, 0x4c, 0x4b, 0xea, - 0x7f, 0x2c, 0xda, 0x75, 0x68, 0xc5, 0x57, 0x31, 0x23, 0xde, 0x91, 0x2a, 0xdd, 0xee, 0x9b, 0x96, - 0xee, 0xab, 0x0a, 0xb4, 0xb2, 0x90, 0x16, 0xe7, 0xbc, 0x34, 0x88, 0xe4, 0xd0, 0x79, 0x07, 0x5a, - 0xa1, 0x0c, 0x95, 0x48, 0xa8, 0xed, 0x83, 0x9e, 0x8a, 0x22, 0xa5, 0x60, 0x4a, 0x4f, 0xad, 0x34, - 0x78, 0xea, 0x22, 0x1a, 0x74, 0x12, 0xf2, 0x6e, 0x68, 0x88, 0x6e, 0xb8, 0x0d, 0x8d, 0x17, 0x96, - 0x3d, 0x41, 0x28, 0xfc, 0xc0, 0x0e, 0x15, 0xab, 0x62, 0xac, 0x7a, 0xc4, 0x0b, 0xa2, 0x2b, 0x81, - 0x46, 0x37, 0xbf, 0xc7, 0x79, 0x23, 0x73, 0xa4, 0x92, 0x7b, 0x0b, 0x5b, 0x21, 0x8d, 0x22, 0xcd, - 0xed, 0xcc, 0x98, 0x32, 0x6e, 0x40, 0xc3, 0x93, 0xf6, 0x55, 0xb7, 0xa4, 0x70, 0x95, 0x57, 0xf3, - 0x11, 0x6c, 0xcb, 0x71, 0xbd, 0x74, 0x28, 0xcf, 0x0c, 0x34, 0x19, 0xa1, 0x20, 0xc5, 0xdc, 0x85, - 0x9d, 0x19, 0x13, 0xaa, 0x37, 0x56, 0xa1, 0xfb, 0xe4, 0x9c, 0x60, 0xf1, 0x29, 0xa3, 0xe6, 0xdf, - 0x15, 0xa8, 0x89, 0x1d, 0x1e, 0x2e, 0x47, 0xa2, 0x1c, 0x48, 0x67, 0xf3, 0xec, 0x77, 0x4b, 0x89, - 0xd0, 0xf3, 0x80, 0x6a, 0xa5, 0x09, 0x2b, 0x0b, 0x02, 0x83, 0x56, 0x59, 0x12, 0x25, 0x31, 0x9b, - 0xa3, 0x22, 0x77, 0xad, 0x05, 0xdc, 0x15, 0x87, 0x0d, 0x2c, 0x1a, 0x36, 0x7f, 0x56, 0xa0, 0xf3, - 0x2d, 0x61, 0x17, 0x41, 0x74, 0xc6, 0x33, 0x14, 0x97, 0xba, 0x1b, 0xbb, 0x20, 0xba, 0x1c, 0x8e, - 0xae, 0x18, 0x56, 0x8c, 0x48, 0x25, 0x8f, 0x07, 0x77, 0x8e, 0x2c, 0xd9, 0xd3, 0x9a, 0xd8, 0xc3, - 0xb2, 0x7e, 0x79, 0x39, 0xc4, 0x22, 0x0d, 0x22, 0xd9, 0xe6, 0x42, 0x0c, 0xb7, 0x9c, 0x28, 0x08, - 0x43, 0x22, 0x23, 0xd5, 0xb9, 0xb1, 0x93, 0xd4, 0x58, 0x3d, 0x95, 0xc2, 0x9d, 0x50, 0x19, 0x6b, - 0xa4, 0xc6, 0x4e, 0x32, 0x63, 0xcd, 0x9c, 0x58, 0x6a, 0xac, 0x25, 0x4a, 0xca, 0x83, 0xe6, 0x61, - 0x98, 0x9c, 0xc6, 0xd6, 0x98, 0xf0, 0x41, 0xc3, 0x02, 0x66, 0xb9, 0xc3, 0x84, 0x2f, 0x05, 0x74, - 0xdd, 0xd8, 0x84, 0x4e, 0x48, 0x22, 0x2c, 0x4a, 0xb5, 0x5b, 0x45, 0xa2, 0x74, 0xe3, 0x1a, 0x6c, - 0x88, 0xe5, 0x90, 0xfa, 0xc3, 0x33, 0x12, 0xf9, 0xc4, 0xf5, 0x02, 0x87, 0xa8, 0x38, 0x76, 0x61, - 0x3d, 0x3b, 0xe4, 0xad, 0x2e, 0x8e, 0x44, 0x3c, 0xe6, 0x09, 0xf4, 0x4e, 0x26, 0x78, 0x1b, 0x61, - 0x2e, 0xf5, 0xc7, 0x8f, 0x2d, 0x66, 0x19, 0xab, 0x98, 0x27, 0x12, 0xd1, 0xc0, 0x89, 0x95, 0x43, - 0xd4, 0x66, 0x52, 0x84, 0x38, 0xc3, 0xf4, 0x48, 0x92, 0x86, 0x9f, 0x93, 0xe9, 0x11, 0xa3, 0x9e, - 0x72, 0x68, 0xfe, 0x28, 0x82, 0x90, 0xc4, 0x9b, 0xf8, 0xe1, 0xce, 0xc0, 0xca, 0x0f, 0xf7, 0x6a, - 0x9a, 0xaf, 0x34, 0xd0, 0x7d, 0x58, 0x65, 0x19, 0x8a, 0x21, 0x56, 0xad, 0xa5, 0x1a, 0x63, 0x4b, - 0x49, 0x16, 0x31, 0x9a, 0x5f, 0x00, 0xbc, 0x10, 0x7d, 0x28, 0x10, 0xe3, 0xa8, 0xc9, 0x13, 0x84, - 0x44, 0x7b, 0xd6, 0x65, 0xc6, 0x0e, 0xdf, 0xc2, 0x98, 0x7e, 0xb2, 0xa8, 0x6b, 0xab, 0x7b, 0x8a, - 0x6e, 0xfe, 0x53, 0x81, 0xb6, 0xb4, 0x20, 0x41, 0xa2, 0x09, 0x1b, 0x7b, 0x2f, 0x35, 0xb1, 0x97, - 0x5a, 0x2c, 0x7e, 0xcc, 0x72, 0x3e, 0xb1, 0x0c, 0xe3, 0x0b, 0x2b, 0x54, 0x5e, 0xb4, 0x45, 0x62, - 0xb7, 0xa1, 0x23, 0xb3, 0xa1, 0x04, 0xf5, 0x45, 0x82, 0x77, 0xf9, 0xb8, 0x44, 0x24, 0x62, 0xba, - 0xb6, 0x0f, 0xae, 0x17, 0x24, 0x04, 0xc6, 0x7d, 0xf1, 0xfb, 0xc4, 0x67, 0xd1, 0xd5, 0xe0, 0x2e, - 0xc0, 0x74, 0xc5, 0xdb, 0xee, 0x8c, 0x5c, 0xa9, 0xca, 0xc6, 0x48, 0xce, 0x2d, 0x37, 0x51, 0x91, - 0x3f, 0xac, 0x3e, 0xa8, 0x98, 0xdf, 0xc0, 0xea, 0x57, 0xee, 0x19, 0x0d, 0x72, 0x2a, 0x28, 0xe5, - 0x59, 0x3f, 0x07, 0x91, 0x8a, 0x97, 0x2f, 0xa9, 0x8f, 0x4b, 0x49, 0x17, 0xf6, 0x7d, 0x10, 0x4e, - 0x6f, 0x74, 0xd2, 0x9e, 0xac, 0x97, 0xbf, 0x34, 0x80, 0xa9, 0x31, 0xe3, 0x21, 0x0c, 0x68, 0x30, - 0xc4, 0x92, 0x3a, 0xa7, 0x36, 0x91, 0x2d, 0x30, 0x8c, 0x88, 0x9d, 0x44, 0x31, 0x3d, 0x27, 0x6a, - 0xfe, 0x6d, 0xab, 0x58, 0xca, 0x18, 0x3e, 0x82, 0xad, 0xa9, 0xae, 0x93, 0x53, 0xab, 0x2e, 0x55, - 0xbb, 0x0f, 0x1b, 0xa8, 0x86, 0x83, 0x2b, 0x29, 0x28, 0x69, 0x4b, 0x95, 0x3e, 0x85, 0xdd, 0x1c, - 0x4e, 0x5e, 0xa9, 0x39, 0x55, 0x7d, 0xa9, 0xea, 0xc7, 0xb0, 0x8d, 0xaa, 0x17, 0x16, 0x65, 0x65, - 0xbd, 0xda, 0x7f, 0xc0, 0xe9, 0x91, 0x68, 0x5c, 0xc0, 0x59, 0x5f, 0xaa, 0xf4, 0x01, 0xac, 0xa3, - 0x52, 0xc9, 0x4f, 0xe3, 0x75, 0x2a, 0x31, 0xb1, 0x19, 0x4e, 0x95, 0x9c, 0x4a, 0x73, 0x99, 0x0a, - 0x7e, 0x5e, 0x3a, 0xcf, 0x92, 0x31, 0x61, 0xee, 0x28, 0xab, 0xfe, 0x37, 0x6d, 0xa0, 0x57, 0x55, - 0x68, 0x1f, 0x8e, 0xa3, 0x20, 0x09, 0x0b, 0x5d, 0x2e, 0x6b, 0x78, 0xa6, 0xcb, 0xa5, 0xcc, 0x1d, - 0xe8, 0xc8, 0xaf, 0xa7, 0x12, 0x93, 0xcd, 0x65, 0xcc, 0x96, 0x3a, 0xbf, 0x23, 0x8d, 0x38, 0x66, - 0x25, 0x58, 0x6c, 0xaf, 0x5c, 0xf9, 0x7d, 0x06, 0xdd, 0x89, 0x0c, 0x44, 0x49, 0xca, 0x54, 0xde, - 0x4a, 0x3d, 0x4f, 0x01, 0xee, 0xe7, 0x03, 0x96, 0x4d, 0xf4, 0x0c, 0xd6, 0x67, 0x36, 0x8b, 0xbd, - 0x64, 0xe6, 0x7b, 0xa9, 0x7d, 0xb0, 0xa1, 0xcc, 0xe6, 0xb5, 0x44, 0x83, 0x85, 0x50, 0x93, 0x78, - 0xde, 0x87, 0xae, 0x2f, 0x3f, 0x3a, 0x19, 0x13, 0x5a, 0x4e, 0xb1, 0xf0, 0x41, 0x42, 0x36, 0x6c, - 0x81, 0x6f, 0x2e, 0x1b, 0x79, 0x6e, 0x31, 0x1f, 0xbc, 0x22, 0x50, 0xcc, 0x0b, 0x15, 0xfd, 0xea, - 0x72, 0x38, 0xef, 0x59, 0x72, 0xf0, 0x5b, 0x1d, 0xb4, 0x47, 0x47, 0x5f, 0x1b, 0x2f, 0x61, 0xb5, - 0xf4, 0x98, 0x32, 0xd2, 0xb1, 0x32, 0xff, 0xc5, 0x37, 0x78, 0x7b, 0xd1, 0xb1, 0xba, 0x38, 0xac, - 0x70, 0x9b, 0xa5, 0x5b, 0x45, 0x66, 0x73, 0xfe, 0x85, 0x25, 0xb3, 0xb9, 0xe8, 0x32, 0xb2, 0x62, - 0x7c, 0x02, 0x75, 0xf9, 0xf4, 0x32, 0x36, 0x95, 0x6c, 0xe1, 0x0d, 0x37, 0xd8, 0x2a, 0xed, 0x66, - 0x8a, 0xcf, 0xa1, 0x5b, 0x78, 0xd4, 0x1a, 0xd7, 0x0a, 0xbe, 0x8a, 0x2f, 0xb7, 0xc1, 0x5b, 0xf3, - 0x0f, 0x33, 0x6b, 0x87, 0x00, 0xd3, 0x27, 0x90, 0xd1, 0x57, 0xd2, 0x33, 0x2f, 0xc0, 0xc1, 0xee, - 0x9c, 0x93, 0xcc, 0xc8, 0x29, 0xac, 0x95, 0xdf, 0x38, 0x46, 0x89, 0xd5, 0xf2, 0x8b, 0x64, 0x70, - 0x63, 0xe1, 0x79, 0xde, 0x6c, 0xf9, 0xa5, 0x93, 0x99, 0x5d, 0xf0, 0x6e, 0xca, 0xcc, 0x2e, 0x7c, - 0x22, 0xad, 0x18, 0xdf, 0x41, 0xaf, 0xf8, 0x48, 0x31, 0x52, 0x92, 0xe6, 0xbe, 0x9d, 0x06, 0xd7, - 0x17, 0x9c, 0x66, 0x06, 0x3f, 0x94, 0x8d, 0x80, 0x37, 0x97, 0x34, 0x67, 0xb9, 0x17, 0xcc, 0x60, - 0xb3, 0xb8, 0x99, 0x69, 0xdd, 0x83, 0xba, 0xbc, 0x8f, 0x66, 0x05, 0x50, 0xb8, 0x9e, 0x0e, 0x3a, - 0xf9, 0x5d, 0x73, 0xe5, 0x5e, 0x05, 0x67, 0x5e, 0xf3, 0x29, 0x61, 0xb2, 0x3b, 0xf2, 0xae, 0x66, - 0x54, 0xc4, 0x26, 0x57, 0x19, 0xd5, 0xc5, 0x7f, 0x2e, 0xf7, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, - 0x66, 0x65, 0xcd, 0xa0, 0x80, 0x11, 0x00, 0x00, + // 1542 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x6e, 0x1c, 0x45, + 0x17, 0xf6, 0xec, 0x33, 0x67, 0x16, 0xdb, 0xed, 0x6d, 0x3c, 0xf9, 0xf3, 0xc7, 0x74, 0x02, 0x89, + 0x50, 0x64, 0x05, 0x87, 0x25, 0x04, 0x09, 0x08, 0x4e, 0x44, 0x40, 0x09, 0x58, 0xb1, 0x8d, 0xc4, + 0x0d, 0xa3, 0x9e, 0xee, 0x62, 0xa6, 0x71, 0x6f, 0x74, 0x55, 0x7b, 0x79, 0x85, 0x70, 0xcf, 0x5b, + 0x20, 0x71, 0xc5, 0x03, 0xf0, 0x04, 0x3c, 0x07, 0x4f, 0xc1, 0xa9, 0xa5, 0x6b, 0xba, 0x7b, 0x16, + 0x93, 0x0b, 0x6e, 0x46, 0xaa, 0xaa, 0xb3, 0x7c, 0xe7, 0x3b, 0x4b, 0x57, 0x0d, 0xb4, 0xac, 0xc8, + 0xdd, 0x8f, 0xe2, 0x90, 0x85, 0x46, 0x8d, 0x5d, 0x45, 0x84, 0x9a, 0x23, 0xd8, 0x3c, 0x8d, 0x1c, + 0x8b, 0x91, 0xa3, 0x38, 0xb4, 0x09, 0xa5, 0xaf, 0xc8, 0xcf, 0x09, 0xa1, 0xcc, 0x00, 0x28, 0xbb, + 0x4e, 0xbf, 0xb4, 0x57, 0xba, 0xd7, 0x32, 0xda, 0x50, 0x89, 0x70, 0x51, 0x16, 0x0b, 0x3c, 0xb1, + 0xbd, 0x90, 0x92, 0x63, 0xe6, 0xb8, 0x41, 0xbf, 0x82, 0x7b, 0x4d, 0xa3, 0x0b, 0xb5, 0x0b, 0xd7, + 0x61, 0x93, 0x7e, 0x15, 0x97, 0x5d, 0xa3, 0x07, 0xf5, 0x09, 0x71, 0xc7, 0x13, 0xd6, 0xaf, 0xf1, + 0xb5, 0xb9, 0x03, 0x5b, 0x05, 0x1f, 0x34, 0x0a, 0x03, 0x4a, 0xcc, 0x5f, 0x4a, 0xb0, 0x7d, 0x18, + 0x13, 0x3c, 0x39, 0x0c, 0x03, 0x66, 0xb9, 0x01, 0x89, 0xe7, 0xf9, 0xc7, 0xc5, 0x28, 0x09, 0x1c, + 0x8f, 0x1c, 0x59, 0xe8, 0x63, 0x0a, 0x63, 0x42, 0xec, 0xb3, 0x28, 0x74, 0x03, 0x26, 0x60, 0xb4, + 0x38, 0x0c, 0x2a, 0x50, 0x55, 0xc5, 0x12, 0x61, 0xe0, 0x32, 0x4c, 0x24, 0x8c, 0x74, 0x4d, 0xe2, + 0xb8, 0x5f, 0x4f, 0xd7, 0x9e, 0x35, 0x22, 0x1e, 0xed, 0x37, 0xf6, 0x2a, 0xf7, 0x5a, 0xe6, 0xa7, + 0xb0, 0x33, 0x03, 0x46, 0x02, 0x35, 0x6e, 0x43, 0xcb, 0x4e, 0x37, 0x05, 0xa8, 0xf6, 0xc1, 0xda, + 0xbe, 0x20, 0x70, 0x5f, 0x0b, 0x9b, 0x8f, 0xa0, 0x7b, 0xec, 0x8e, 0x03, 0xcb, 0xbb, 0x96, 0x43, + 0x8e, 0x44, 0x48, 0x0a, 0xe0, 0x5d, 0x73, 0x0d, 0x7a, 0xa9, 0xa6, 0x62, 0xe6, 0xf7, 0x12, 0xac, + 0x3f, 0x71, 0x9c, 0x25, 0x49, 0x59, 0x83, 0x26, 0x23, 0xb1, 0xef, 0x72, 0x2b, 0x65, 0x91, 0x85, + 0x5d, 0xa8, 0x26, 0x14, 0xf1, 0x55, 0x04, 0xbe, 0xb6, 0xc2, 0x77, 0x8a, 0x5b, 0x46, 0x07, 0xaa, + 0x56, 0x3c, 0xa6, 0x48, 0x4c, 0x45, 0x62, 0x21, 0xc1, 0x39, 0xb2, 0xa2, 0x16, 0xf6, 0x85, 0xa3, + 0x28, 0x51, 0x28, 0x1b, 0x79, 0x3a, 0x9b, 0x05, 0x3a, 0x5b, 0x05, 0x3a, 0x81, 0xaf, 0x31, 0xfc, + 0xaa, 0xf0, 0x85, 0x36, 0x12, 0x85, 0xb2, 0xcb, 0x17, 0x63, 0x15, 0x76, 0xd7, 0xd8, 0x86, 0x9e, + 0xe5, 0x38, 0x2e, 0x73, 0x43, 0x04, 0xfd, 0xa5, 0xeb, 0x50, 0x84, 0x5a, 0xc1, 0xf0, 0x37, 0xc1, + 0xc8, 0xc6, 0xaa, 0x28, 0x78, 0xa1, 0xd3, 0xa1, 0xf3, 0x3c, 0x8f, 0x87, 0xb7, 0x73, 0x85, 0x50, + 0x16, 0xb1, 0xaf, 0xa7, 0xb9, 0xd1, 0x07, 0xe6, 0x00, 0xfa, 0xb3, 0xd6, 0x94, 0xa7, 0x87, 0xb0, + 0xf3, 0x94, 0x78, 0xe4, 0x3a, 0x4f, 0x48, 0x62, 0x60, 0xf9, 0x44, 0xe6, 0x90, 0x1b, 0x9c, 0x55, + 0x52, 0x06, 0x6f, 0xc3, 0xd6, 0x0b, 0x97, 0xb2, 0xa5, 0xe6, 0xcc, 0xef, 0x01, 0xa6, 0x02, 0xda, + 0xb8, 0x76, 0x45, 0x2e, 0x5d, 0xa6, 0x12, 0x8b, 0x24, 0x32, 0x3b, 0x52, 0xbd, 0xb6, 0x01, 0xed, + 0x24, 0x70, 0x2f, 0x8f, 0x43, 0xfb, 0x8c, 0x30, 0x2a, 0x4a, 0x5d, 0x34, 0x20, 0x9d, 0x10, 0xcf, + 0x13, 0x95, 0xde, 0x34, 0x3f, 0x87, 0xed, 0xa2, 0x7f, 0x55, 0xc8, 0xef, 0x40, 0x7b, 0xca, 0x16, + 0x45, 0x6f, 0x95, 0x45, 0x74, 0x75, 0x8e, 0x19, 0xb2, 0x35, 0x0f, 0xf8, 0x1e, 0xf4, 0x74, 0xd1, + 0x0b, 0x21, 0x59, 0x0a, 0x16, 0x4b, 0xa8, 0x92, 0xf8, 0xad, 0x04, 0x0d, 0x95, 0xce, 0xb4, 0xa4, + 0xfe, 0xc3, 0xa2, 0x5d, 0x87, 0x16, 0xbd, 0xa2, 0x8c, 0xf8, 0x47, 0xaa, 0x74, 0xbb, 0x6f, 0x5a, + 0xba, 0xaf, 0x4b, 0xd0, 0xd2, 0x21, 0x2d, 0xce, 0x79, 0x61, 0x10, 0xc9, 0xa1, 0xf3, 0x16, 0xb4, + 0x22, 0x19, 0x2a, 0x91, 0x50, 0xdb, 0x07, 0x3d, 0x15, 0x45, 0x4a, 0xc1, 0x94, 0x9e, 0x5a, 0x61, + 0xf0, 0xd4, 0x45, 0x34, 0xe8, 0x24, 0xe2, 0xdd, 0xd0, 0x10, 0xdd, 0x70, 0x17, 0x1a, 0x2f, 0x2d, + 0x7b, 0x82, 0x50, 0xf8, 0x81, 0x1d, 0x29, 0x56, 0xc5, 0x58, 0xf5, 0x89, 0x1f, 0xc6, 0x57, 0x02, + 0x4d, 0xd5, 0xfc, 0x0e, 0xe7, 0x8d, 0xcc, 0x91, 0x4a, 0xee, 0x1d, 0x6c, 0x85, 0x34, 0x8a, 0x34, + 0xb7, 0x33, 0x63, 0xca, 0xb8, 0x05, 0x0d, 0x5f, 0xda, 0x57, 0xdd, 0x92, 0xc2, 0x55, 0x5e, 0xcd, + 0x27, 0xb0, 0x2d, 0xc7, 0xf5, 0xd2, 0xa1, 0x3c, 0x33, 0xd0, 0x64, 0x84, 0x82, 0x14, 0x73, 0x17, + 0x76, 0x66, 0x4c, 0xa8, 0xde, 0x58, 0x85, 0xee, 0xb3, 0x73, 0x82, 0xc5, 0xa7, 0x8c, 0x9a, 0x7f, + 0x95, 0xa0, 0x26, 0x76, 0x78, 0xb8, 0x1c, 0x89, 0x72, 0x20, 0x9d, 0xcd, 0xb3, 0xdf, 0x2d, 0x24, + 0xa2, 0x9a, 0x05, 0x54, 0x2b, 0x4c, 0x58, 0x59, 0x10, 0x18, 0xb4, 0xca, 0x92, 0x28, 0x89, 0xd9, + 0x1c, 0xe5, 0xb9, 0x6b, 0x2d, 0xe0, 0x2e, 0x3f, 0x6c, 0x60, 0xd1, 0xb0, 0xf9, 0xa3, 0x04, 0x9d, + 0x6f, 0x08, 0xbb, 0x08, 0xe3, 0x33, 0x9e, 0x21, 0x5a, 0xe8, 0x6e, 0xec, 0x82, 0xf8, 0x72, 0x38, + 0xba, 0x62, 0x58, 0x31, 0x22, 0x95, 0x3c, 0x1e, 0xdc, 0x39, 0xb2, 0x64, 0x4f, 0x57, 0xc4, 0x1e, + 0x96, 0xf5, 0xab, 0xcb, 0x21, 0x16, 0x69, 0x18, 0xcb, 0x36, 0x17, 0x62, 0xb8, 0xe5, 0xc4, 0x61, + 0x14, 0x11, 0x19, 0x69, 0x95, 0x1b, 0x3b, 0x49, 0x8d, 0xd5, 0x53, 0x29, 0xdc, 0x89, 0x94, 0xb1, + 0x46, 0x6a, 0xec, 0x44, 0x1b, 0x6b, 0x66, 0xc4, 0x52, 0x63, 0x2d, 0x51, 0x52, 0x3e, 0x34, 0x0f, + 0xa3, 0xe4, 0x94, 0x5a, 0x63, 0xc2, 0x07, 0x0d, 0x0b, 0x99, 0xe5, 0x0d, 0x13, 0xbe, 0x14, 0xd0, + 0xab, 0xc6, 0x26, 0x74, 0x22, 0x12, 0x63, 0x51, 0xaa, 0xdd, 0x32, 0x12, 0x55, 0x35, 0x6e, 0xc0, + 0x86, 0x58, 0x0e, 0xdd, 0x60, 0x78, 0x46, 0xe2, 0x80, 0x78, 0x7e, 0xe8, 0x10, 0x15, 0xc7, 0x2e, + 0xac, 0xeb, 0x43, 0xde, 0xea, 0xe2, 0x48, 0xc4, 0x63, 0x9e, 0x40, 0xef, 0x64, 0x82, 0xb7, 0x11, + 0xe6, 0xb9, 0xc1, 0xf8, 0xa9, 0xc5, 0x2c, 0x63, 0x15, 0xf3, 0x44, 0x62, 0x37, 0x74, 0xa8, 0x72, + 0x88, 0xda, 0x4c, 0x8a, 0x10, 0x67, 0x98, 0x1e, 0x49, 0xd2, 0xf0, 0x73, 0x32, 0x3d, 0x62, 0xae, + 0xaf, 0x1c, 0x9a, 0x3f, 0x88, 0x20, 0x24, 0xf1, 0x26, 0x7e, 0xb8, 0x35, 0x58, 0xf9, 0xe1, 0x5e, + 0x4d, 0xf3, 0x95, 0x06, 0xba, 0x0f, 0xab, 0x4c, 0xa3, 0x18, 0x62, 0xd5, 0x5a, 0xaa, 0x31, 0xb6, + 0x94, 0x64, 0x1e, 0xa3, 0xf9, 0x19, 0xc0, 0x4b, 0xd1, 0x87, 0x02, 0x31, 0x8e, 0x9a, 0x2c, 0x41, + 0x48, 0xb4, 0x6f, 0x5d, 0x6a, 0x76, 0xf8, 0x16, 0xc6, 0xf4, 0xa3, 0xe5, 0x7a, 0xb6, 0xba, 0xa7, + 0x54, 0xcd, 0xbf, 0x4b, 0xd0, 0x96, 0x16, 0x24, 0x48, 0x34, 0x61, 0x63, 0xef, 0xa5, 0x26, 0xf6, + 0x52, 0x8b, 0xf9, 0x8f, 0x59, 0xc6, 0x27, 0x96, 0x21, 0xbd, 0xb0, 0x22, 0xe5, 0xa5, 0xb2, 0x48, + 0xec, 0x2e, 0x74, 0x64, 0x36, 0x94, 0x60, 0x75, 0x91, 0xe0, 0x7d, 0x3e, 0x2e, 0x11, 0x89, 0x98, + 0xae, 0xed, 0x83, 0x9b, 0x39, 0x09, 0x81, 0x71, 0x5f, 0xfc, 0x3e, 0x0b, 0x58, 0x7c, 0x35, 0xb8, + 0x0f, 0x30, 0x5d, 0xf1, 0xb6, 0x3b, 0x23, 0x57, 0xaa, 0xb2, 0x31, 0x92, 0x73, 0xcb, 0x4b, 0x54, + 0xe4, 0x8f, 0xcb, 0x8f, 0x4a, 0xe6, 0xd7, 0xb0, 0xfa, 0x85, 0x77, 0xe6, 0x86, 0x19, 0x15, 0x94, + 0xf2, 0xad, 0x9f, 0xc2, 0x58, 0xc5, 0xcb, 0x97, 0x6e, 0x80, 0x4b, 0x49, 0x17, 0xf6, 0x7d, 0x18, + 0x4d, 0x6f, 0x74, 0xd2, 0x9e, 0xac, 0x97, 0x3f, 0x2b, 0x00, 0x53, 0x63, 0xc6, 0x63, 0x18, 0xb8, + 0xe1, 0x10, 0x4b, 0xea, 0xdc, 0xb5, 0x89, 0x6c, 0x81, 0x61, 0x4c, 0xec, 0x24, 0xa6, 0xee, 0x39, + 0x51, 0xf3, 0x6f, 0x5b, 0xc5, 0x52, 0xc4, 0xf0, 0x01, 0x6c, 0x4d, 0x75, 0x9d, 0x8c, 0x5a, 0x79, + 0xa9, 0xda, 0x43, 0xd8, 0x40, 0x35, 0x1c, 0x5c, 0x49, 0x4e, 0xa9, 0xb2, 0x54, 0xe9, 0x63, 0xd8, + 0xcd, 0xe0, 0xe4, 0x95, 0x9a, 0x51, 0xad, 0x2e, 0x55, 0xfd, 0x10, 0xb6, 0x51, 0xf5, 0xc2, 0x72, + 0x59, 0x51, 0xaf, 0xf6, 0x2f, 0x70, 0xfa, 0x24, 0x1e, 0xe7, 0x70, 0xd6, 0x97, 0x2a, 0xbd, 0x07, + 0xeb, 0xa8, 0x54, 0xf0, 0xd3, 0xb8, 0x4e, 0x85, 0x12, 0x9b, 0xe1, 0x54, 0xc9, 0xa8, 0x34, 0x97, + 0xa9, 0xe0, 0xe7, 0xa5, 0xf3, 0x3c, 0x19, 0x13, 0xe6, 0x8d, 0x74, 0xf5, 0xbf, 0x69, 0x03, 0xbd, + 0x2e, 0x43, 0xfb, 0x70, 0x1c, 0x87, 0x49, 0x94, 0xeb, 0x72, 0x59, 0xc3, 0x33, 0x5d, 0x2e, 0x65, + 0xee, 0x41, 0x47, 0x7e, 0x3d, 0x95, 0x98, 0x6c, 0x2e, 0x63, 0xb6, 0xd4, 0xf9, 0x1d, 0x69, 0xc4, + 0x31, 0x2b, 0xc1, 0x7c, 0x7b, 0x65, 0xca, 0xef, 0x13, 0xe8, 0x4e, 0x64, 0x20, 0x4a, 0x52, 0xa6, + 0xf2, 0x4e, 0xea, 0x79, 0x0a, 0x70, 0x3f, 0x1b, 0xb0, 0x6c, 0xa2, 0xe7, 0xb0, 0x3e, 0xb3, 0x99, + 0xef, 0x25, 0x33, 0xdb, 0x4b, 0xed, 0x83, 0x0d, 0x65, 0x36, 0xab, 0x25, 0x1a, 0xec, 0x52, 0x5e, + 0x03, 0xf4, 0xc5, 0xd9, 0x78, 0x17, 0xba, 0x81, 0xfc, 0xf8, 0x68, 0x46, 0x2a, 0x19, 0x03, 0xb9, + 0x0f, 0x13, 0xb2, 0x62, 0x0b, 0x9c, 0x73, 0x59, 0xc9, 0x72, 0x8c, 0x79, 0xe1, 0x95, 0x81, 0x62, + 0x7e, 0xa4, 0xd2, 0xa0, 0x2e, 0x89, 0xf3, 0x9e, 0x27, 0x07, 0xbf, 0xd6, 0xa1, 0xf2, 0xe4, 0xe8, + 0x2b, 0xe3, 0x15, 0xac, 0x16, 0x1e, 0x55, 0x46, 0x3a, 0x5e, 0xe6, 0xbf, 0xfc, 0x06, 0xff, 0x5f, + 0x74, 0xac, 0x2e, 0x10, 0x2b, 0xdc, 0x66, 0xe1, 0x76, 0xa1, 0x6d, 0xce, 0xbf, 0xb8, 0x68, 0x9b, + 0x8b, 0x2e, 0x25, 0x2b, 0xc6, 0x47, 0x50, 0x97, 0x4f, 0x30, 0x63, 0x53, 0xc9, 0xe6, 0xde, 0x72, + 0x83, 0xad, 0xc2, 0xae, 0x56, 0x7c, 0x01, 0xdd, 0xdc, 0xe3, 0xd6, 0xb8, 0x91, 0xf3, 0x95, 0x7f, + 0xc1, 0x0d, 0xfe, 0x37, 0xff, 0x50, 0x5b, 0x3b, 0x04, 0x98, 0x3e, 0x85, 0x8c, 0xbe, 0x92, 0x9e, + 0x79, 0x09, 0x0e, 0x76, 0xe7, 0x9c, 0x68, 0x23, 0xa7, 0xb0, 0x56, 0x7c, 0xeb, 0x18, 0x05, 0x56, + 0x8b, 0x2f, 0x93, 0xc1, 0xad, 0x85, 0xe7, 0x59, 0xb3, 0xc5, 0x17, 0x8f, 0x36, 0xbb, 0xe0, 0xfd, + 0xa4, 0xcd, 0x2e, 0x7c, 0x2a, 0xad, 0x18, 0xdf, 0x42, 0x2f, 0xff, 0x58, 0x31, 0x52, 0x92, 0xe6, + 0xbe, 0xa1, 0x06, 0x37, 0x17, 0x9c, 0x6a, 0x83, 0xef, 0x43, 0x4d, 0x3e, 0x4b, 0xd2, 0x8a, 0xcf, + 0xbe, 0x64, 0x06, 0x9b, 0xf9, 0x4d, 0xad, 0xf5, 0x00, 0xea, 0xf2, 0x5e, 0xaa, 0x0b, 0x20, 0x77, + 0x4d, 0x1d, 0x74, 0xb2, 0xbb, 0xe6, 0xca, 0x83, 0x52, 0xea, 0x87, 0xe6, 0xfc, 0xd0, 0x79, 0x7e, + 0x32, 0xc9, 0x19, 0xd5, 0xc5, 0xdf, 0x2f, 0x0f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x37, + 0x3f, 0x0d, 0x8b, 0x11, 0x00, 0x00, } diff --git a/api/grpc/types/api.proto b/api/grpc/types/api.proto index 75ad7d9..a9fecb4 100644 --- a/api/grpc/types/api.proto +++ b/api/grpc/types/api.proto @@ -13,7 +13,7 @@ service API { rpc ListCheckpoint(ListCheckpointRequest) returns (ListCheckpointResponse) {} rpc State(StateRequest) returns (StateResponse) {} rpc Events(EventsRequest) returns (stream Event) {} - rpc GetStats(StatsRequest) returns (stream Stats) {} + rpc Stats(StatsRequest) returns (StatsResponse) {} } message UpdateProcessRequest { @@ -246,7 +246,7 @@ message CgroupStats { map hugetlb_stats = 4; // the map is in the format "size of hugepage: stats of the hugepage" } -message Stats { +message StatsResponse { repeated NetworkStats network_stats = 1; CgroupStats cgroup_stats = 2; uint64 timestamp = 3; diff --git a/ctr/container.go b/ctr/container.go index a7a7d38..4e2bf98 100644 --- a/ctr/container.go +++ b/ctr/container.go @@ -423,17 +423,15 @@ var statsCommand = cli.Command{ Id: context.Args().First(), } c := getClient(context) - stream, err := c.GetStats(netcontext.Background(), req) + stats, err := c.Stats(netcontext.Background(), req) if err != nil { fatal(err.Error(), 1) } - for { - stats, err := stream.Recv() - if err != nil { - fatal(err.Error(), 1) - } - fmt.Println(stats) + data, err := json.Marshal(stats) + if err != nil { + fatal(err.Error(), 1) } + fmt.Print(string(data)) }, } diff --git a/runtime/container.go b/runtime/container.go index ae0e63c..a8795c4 100644 --- a/runtime/container.go +++ b/runtime/container.go @@ -46,7 +46,8 @@ type Container interface { // Pids returns all pids inside the container Pids() ([]int, error) // Stats returns realtime container stats and resource information - // Stats() (*Stat, error) // OOM signals the channel if the container received an OOM notification + Stats() (*Stat, error) + // OOM signals the channel if the container received an OOM notification // OOM() (<-chan struct{}, error) } @@ -357,17 +358,37 @@ func (c *container) DeleteCheckpoint(name string) error { } func (c *container) Pids() ([]int, error) { - f, err := libcontainer.New(specs.LinuxStateDirectory, libcontainer.Cgroupfs) - if err != nil { - return nil, err - } - container, err := f.Load(c.id) + container, err := c.getLibctContainer() if err != nil { return nil, err } return container.Processes() } +func (c *container) Stats() (*Stat, error) { + container, err := c.getLibctContainer() + if err != nil { + return nil, err + } + now := time.Now() + stats, err := container.Stats() + if err != nil { + return nil, err + } + return &Stat{ + Timestamp: now, + Data: stats, + }, nil +} + +func (c *container) getLibctContainer() (libcontainer.Container, error) { + f, err := libcontainer.New(specs.LinuxStateDirectory, libcontainer.Cgroupfs) + if err != nil { + return nil, err + } + return f.Load(c.id) +} + func getRootIDs(s *specs.LinuxSpec) (int, int, error) { if s == nil { return 0, 0, nil diff --git a/supervisor/event.go b/supervisor/event.go index 654cb96..1397ee5 100644 --- a/supervisor/event.go +++ b/supervisor/event.go @@ -23,8 +23,6 @@ const ( CreateCheckpointEventType EventType = "createCheckpoint" DeleteCheckpointEventType EventType = "deleteCheckpoint" StatsEventType EventType = "events" - UnsubscribeStatsEventType EventType = "unsubscribeStats" - StopStatsEventType EventType = "stopStats" OOMEventType EventType = "oom" ) @@ -59,7 +57,7 @@ type Event struct { Checkpoint *runtime.Checkpoint Err chan error StartResponse chan StartResponse - Stats chan interface{} + Stat chan *runtime.Stat CloseStdin bool ResizeTty bool Width int diff --git a/supervisor/exit.go b/supervisor/exit.go index d1bf276..14ececa 100644 --- a/supervisor/exit.go +++ b/supervisor/exit.go @@ -39,10 +39,6 @@ func (h *ExitEvent) Handle(e *Event) error { ne.Pid = proc.ID() h.s.SendEvent(ne) - // remove stats collection for container - stopCollect := NewEvent(StopStatsEventType) - stopCollect.ID = container.ID() - h.s.SendEvent(stopCollect) ExitProcessTimer.UpdateSince(start) return nil diff --git a/supervisor/metrics.go b/supervisor/metrics.go index 2e1b38a..38e48a7 100644 --- a/supervisor/metrics.go +++ b/supervisor/metrics.go @@ -6,6 +6,7 @@ var ( ContainerCreateTimer = metrics.NewTimer() ContainerDeleteTimer = metrics.NewTimer() ContainerStartTimer = metrics.NewTimer() + ContainerStatsTimer = metrics.NewTimer() ContainersCounter = metrics.NewCounter() EventSubscriberCounter = metrics.NewCounter() EventsCounter = metrics.NewCounter() @@ -19,6 +20,7 @@ func Metrics() map[string]interface{} { "container-create-time": ContainerCreateTimer, "container-delete-time": ContainerDeleteTimer, "container-start-time": ContainerStartTimer, + "container-stats-time": ContainerStatsTimer, "containers": ContainersCounter, "event-subscribers": EventSubscriberCounter, "events": EventsCounter, diff --git a/supervisor/stats.go b/supervisor/stats.go index 7f3a9ec..7ca44a0 100644 --- a/supervisor/stats.go +++ b/supervisor/stats.go @@ -1,40 +1,27 @@ package supervisor +import "time" + type StatsEvent struct { s *Supervisor } -type UnsubscribeStatsEvent struct { - s *Supervisor -} - -type StopStatsEvent struct { - s *Supervisor -} - func (h *StatsEvent) Handle(e *Event) error { + start := time.Now() i, ok := h.s.containers[e.ID] if !ok { return ErrContainerNotFound } - e.Stats = h.s.statsCollector.collect(i.container) - return nil -} - -func (h *UnsubscribeStatsEvent) Handle(e *Event) error { - i, ok := h.s.containers[e.ID] - if !ok { - return ErrContainerNotFound - } - h.s.statsCollector.unsubscribe(i.container, e.Stats) - return nil -} - -func (h *StopStatsEvent) Handle(e *Event) error { - i, ok := h.s.containers[e.ID] - if !ok { - return ErrContainerNotFound - } - h.s.statsCollector.stopCollection(i.container) - return nil + // TODO: use workers for this + go func() { + s, err := i.container.Stats() + if err != nil { + e.Err <- err + return + } + e.Err <- nil + e.Stat <- s + ContainerStatsTimer.UpdateSince(start) + }() + return errDeferedResponse } diff --git a/supervisor/stats_collector.go b/supervisor/stats_collector.go deleted file mode 100644 index 178458b..0000000 --- a/supervisor/stats_collector.go +++ /dev/null @@ -1,241 +0,0 @@ -package supervisor - -import ( - "bufio" - "fmt" - "os" - "strconv" - "strings" - "sync" - "time" - - "github.com/docker/containerd/api/grpc/types" - "github.com/docker/containerd/runtime" - "github.com/docker/docker/pkg/pubsub" - "github.com/opencontainers/runc/libcontainer" - "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/opencontainers/runc/libcontainer/system" -) - -func convertBlkioEntryToPb(b []cgroups.BlkioStatEntry) []*types.BlkioStatsEntry { - var pbEs []*types.BlkioStatsEntry - for _, e := range b { - pbEs = append(pbEs, &types.BlkioStatsEntry{ - Major: e.Major, - Minor: e.Minor, - Op: e.Op, - Value: e.Value, - }) - } - return pbEs -} - -func convertToPb(st *runtime.Stat) *types.Stats { - pbSt := &types.Stats{ - Timestamp: uint64(st.Timestamp.Unix()), - CgroupStats: &types.CgroupStats{}, - } - lcSt, ok := st.Data.(*libcontainer.Stats) - if !ok { - return pbSt - } - cpuSt := lcSt.CgroupStats.CpuStats - pbSt.CgroupStats.CpuStats = &types.CpuStats{ - CpuUsage: &types.CpuUsage{ - TotalUsage: cpuSt.CpuUsage.TotalUsage, - PercpuUsage: cpuSt.CpuUsage.PercpuUsage, - UsageInKernelmode: cpuSt.CpuUsage.UsageInKernelmode, - UsageInUsermode: cpuSt.CpuUsage.UsageInUsermode, - }, - ThrottlingData: &types.ThrottlingData{ - Periods: cpuSt.ThrottlingData.Periods, - ThrottledPeriods: cpuSt.ThrottlingData.ThrottledPeriods, - ThrottledTime: cpuSt.ThrottlingData.ThrottledTime, - }, - } - memSt := lcSt.CgroupStats.MemoryStats - pbSt.CgroupStats.MemoryStats = &types.MemoryStats{ - Cache: memSt.Cache, - Usage: &types.MemoryData{ - Usage: memSt.Usage.Usage, - MaxUsage: memSt.Usage.MaxUsage, - Failcnt: memSt.Usage.Failcnt, - }, - SwapUsage: &types.MemoryData{ - Usage: memSt.SwapUsage.Usage, - MaxUsage: memSt.SwapUsage.MaxUsage, - Failcnt: memSt.SwapUsage.Failcnt, - }, - } - blkSt := lcSt.CgroupStats.BlkioStats - pbSt.CgroupStats.BlkioStats = &types.BlkioStats{ - IoServiceBytesRecursive: convertBlkioEntryToPb(blkSt.IoServiceBytesRecursive), - IoServicedRecursive: convertBlkioEntryToPb(blkSt.IoServicedRecursive), - IoQueuedRecursive: convertBlkioEntryToPb(blkSt.IoQueuedRecursive), - IoServiceTimeRecursive: convertBlkioEntryToPb(blkSt.IoServiceTimeRecursive), - IoWaitTimeRecursive: convertBlkioEntryToPb(blkSt.IoWaitTimeRecursive), - IoMergedRecursive: convertBlkioEntryToPb(blkSt.IoMergedRecursive), - IoTimeRecursive: convertBlkioEntryToPb(blkSt.IoTimeRecursive), - SectorsRecursive: convertBlkioEntryToPb(blkSt.SectorsRecursive), - } - pbSt.CgroupStats.HugetlbStats = make(map[string]*types.HugetlbStats) - for k, st := range lcSt.CgroupStats.HugetlbStats { - pbSt.CgroupStats.HugetlbStats[k] = &types.HugetlbStats{ - Usage: st.Usage, - MaxUsage: st.MaxUsage, - Failcnt: st.Failcnt, - } - } - return pbSt -} - -type statsPair struct { - ct runtime.Container - pub *pubsub.Publisher -} - -func newStatsCollector(interval time.Duration) *statsCollector { - s := &statsCollector{ - interval: interval, - clockTicksPerSecond: uint64(system.GetClockTicks()), - bufReader: bufio.NewReaderSize(nil, 128), - publishers: make(map[string]*statsPair), - } - go s.run() - return s -} - -// statsCollector manages and provides container resource stats -type statsCollector struct { - m sync.Mutex - supervisor *Supervisor - interval time.Duration - clockTicksPerSecond uint64 - publishers map[string]*statsPair - bufReader *bufio.Reader -} - -// collect registers the container with the collector and adds it to -// the event loop for collection on the specified interval returning -// a channel for the subscriber to receive on. -func (s *statsCollector) collect(c runtime.Container) chan interface{} { - s.m.Lock() - defer s.m.Unlock() - publisher, exists := s.publishers[c.ID()] - if !exists { - pub := pubsub.NewPublisher(100*time.Millisecond, 1024) - publisher = &statsPair{ct: c, pub: pub} - s.publishers[c.ID()] = publisher - } - return publisher.pub.Subscribe() -} - -// stopCollection closes the channels for all subscribers and removes -// the container from metrics collection. -func (s *statsCollector) stopCollection(c runtime.Container) { - s.m.Lock() - if publisher, exists := s.publishers[c.ID()]; exists { - publisher.pub.Close() - delete(s.publishers, c.ID()) - } - s.m.Unlock() -} - -// unsubscribe removes a specific subscriber from receiving updates for a container's stats. -func (s *statsCollector) unsubscribe(c runtime.Container, ch chan interface{}) { - s.m.Lock() - publisher := s.publishers[c.ID()] - if publisher != nil { - publisher.pub.Evict(ch) - if publisher.pub.Len() == 0 { - delete(s.publishers, c.ID()) - } - } - s.m.Unlock() -} - -func (s *statsCollector) run() { - type publishersPair struct { - container runtime.Container - publisher *pubsub.Publisher - } - // we cannot determine the capacity here. - // it will grow enough in first iteration - var pairs []*statsPair - - for range time.Tick(s.interval) { - // it does not make sense in the first iteration, - // but saves allocations in further iterations - pairs = pairs[:0] - - s.m.Lock() - for _, publisher := range s.publishers { - // copy pointers here to release the lock ASAP - pairs = append(pairs, publisher) - } - s.m.Unlock() - if len(pairs) == 0 { - continue - } - - /* - for _, pair := range pairs { - stats, err := pair.ct.Stats() - if err != nil { - logrus.Errorf("Error getting stats for container ID %s", pair.ct.ID()) - continue - } - - pair.pub.Publish(convertToPb(stats)) - } - */ - } -} - -const nanoSecondsPerSecond = 1e9 - -// getSystemCPUUsage returns the host system's cpu usage in -// nanoseconds. An error is returned if the format of the underlying -// file does not match. -// -// Uses /proc/stat defined by POSIX. Looks for the cpu -// statistics line and then sums up the first seven fields -// provided. See `man 5 proc` for details on specific field -// information. -func (s *statsCollector) getSystemCPUUsage() (uint64, error) { - var line string - f, err := os.Open("/proc/stat") - if err != nil { - return 0, err - } - defer func() { - s.bufReader.Reset(nil) - f.Close() - }() - s.bufReader.Reset(f) - err = nil - for err == nil { - line, err = s.bufReader.ReadString('\n') - if err != nil { - break - } - parts := strings.Fields(line) - switch parts[0] { - case "cpu": - if len(parts) < 8 { - return 0, fmt.Errorf("bad format of cpu stats") - } - var totalClockTicks uint64 - for _, i := range parts[1:8] { - v, err := strconv.ParseUint(i, 10, 64) - if err != nil { - return 0, fmt.Errorf("error parsing cpu stats") - } - totalClockTicks += v - } - return (totalClockTicks * nanoSecondsPerSecond) / - s.clockTicksPerSecond, nil - } - } - return 0, fmt.Errorf("bad stats format") -} diff --git a/supervisor/supervisor.go b/supervisor/supervisor.go index f4cc8bf..b3b83bd 100644 --- a/supervisor/supervisor.go +++ b/supervisor/supervisor.go @@ -5,7 +5,6 @@ import ( "os" "sort" "sync" - "time" "github.com/Sirupsen/logrus" "github.com/docker/containerd/chanotify" @@ -14,7 +13,6 @@ import ( ) const ( - statsInterval = 1 * time.Second defaultBufferSize = 2048 // size of queue in eventloop ) @@ -32,14 +30,13 @@ func New(stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error) return nil, err } s := &Supervisor{ - stateDir: stateDir, - containers: make(map[string]*containerInfo), - tasks: tasks, - machine: machine, - subscribers: make(map[chan *Event]struct{}), - statsCollector: newStatsCollector(statsInterval), - el: eventloop.NewChanLoop(defaultBufferSize), - monitor: monitor, + stateDir: stateDir, + containers: make(map[string]*containerInfo), + tasks: tasks, + machine: machine, + subscribers: make(map[chan *Event]struct{}), + el: eventloop.NewChanLoop(defaultBufferSize), + monitor: monitor, } if oom { s.notifier = chanotify.New() @@ -64,8 +61,6 @@ func New(stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error) CreateCheckpointEventType: &CreateCheckpointEvent{s}, DeleteCheckpointEventType: &DeleteCheckpointEvent{s}, StatsEventType: &StatsEvent{s}, - UnsubscribeStatsEventType: &UnsubscribeStatsEvent{s}, - StopStatsEventType: &StopStatsEvent{s}, UpdateProcessEventType: &UpdateProcessEvent{s}, } go s.exitHandler() @@ -91,7 +86,6 @@ type Supervisor struct { subscriberLock sync.RWMutex subscribers map[chan *Event]struct{} machine Machine - statsCollector *statsCollector notifier *chanotify.Notifier el eventloop.EventLoop monitor *Monitor