vbatts/containerd
1
0
Fork 0
Code Pull requests Releases Activity

Implement stats for containerd

Browse source 
This is a single endpoint that clients will poll

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
This commit is contained in:
Michael Crosby 2016-02-11 14:36:32 -08:00
parent 532697f32f
commit bdeb87a090
11 changed files with 265 additions and 493 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
api/grpc/server/server.go
View file

@ -8,10 +8,11 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/codes" "google.golang.org/grpc/codes"
"github.com/Sirupsen/logrus"
"github.com/docker/containerd/api/grpc/types" "github.com/docker/containerd/api/grpc/types"
"github.com/docker/containerd/runtime" "github.com/docker/containerd/runtime"
"github.com/docker/containerd/supervisor" "github.com/docker/containerd/supervisor"
"github.com/opencontainers/runc/libcontainer"
"github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/opencontainers/specs" "github.com/opencontainers/specs"
"golang.org/x/net/context" "golang.org/x/net/context"
) )
@ -297,38 +298,87 @@ func (s *apiServer) Events(r *types.EventsRequest, stream types.API_EventsServer
return nil return nil
} }
func (s *apiServer) GetStats(r *types.StatsRequest, stream types.API_GetStatsServer) error { func (s *apiServer) Stats(ctx context.Context, r *types.StatsRequest) (*types.StatsResponse, error) {
e := supervisor.NewEvent(supervisor.StatsEventType) e := supervisor.NewEvent(supervisor.StatsEventType)
e.ID = r.Id e.ID = r.Id
e.Stat = make(chan *runtime.Stat, 1)
s.sv.SendEvent(e) s.sv.SendEvent(e)
if err := <-e.Err; err != nil { if err := <-e.Err; err != nil {
if err == supervisor.ErrContainerNotFound { return nil, err
return grpc.Errorf(codes.NotFound, err.Error())
}
return err
} }
defer func() { stats := <-e.Stat
ue := supervisor.NewEvent(supervisor.UnsubscribeStatsEventType) t := convertToPb(stats)
ue.ID = e.ID return t, nil
ue.Stats = e.Stats }
s.sv.SendEvent(ue)
if err := <-ue.Err; err != nil { func convertToPb(st *runtime.Stat) *types.StatsResponse {
logrus.Errorf("Error unsubscribing %s: %v", r.Id, err) pbSt := &types.StatsResponse{
} Timestamp: uint64(st.Timestamp.Unix()),
}() CgroupStats: &types.CgroupStats{},
for { }
select { lcSt, ok := st.Data.(*libcontainer.Stats)
case st := <-e.Stats: if !ok {
pbSt, ok := st.(*types.Stats) return pbSt
if !ok { }
panic("invalid stats type from collector") cpuSt := lcSt.CgroupStats.CpuStats
} pbSt.CgroupStats.CpuStats = &types.CpuStats{
if err := stream.Send(pbSt); err != nil { CpuUsage: &types.CpuUsage{
return err TotalUsage: cpuSt.CpuUsage.TotalUsage,
} PercpuUsage: cpuSt.CpuUsage.PercpuUsage,
case <-stream.Context().Done(): UsageInKernelmode: cpuSt.CpuUsage.UsageInKernelmode,
return nil UsageInUsermode: cpuSt.CpuUsage.UsageInUsermode,
} },
} ThrottlingData: &types.ThrottlingData{
return nil Periods: cpuSt.ThrottlingData.Periods,
ThrottledPeriods: cpuSt.ThrottlingData.ThrottledPeriods,
ThrottledTime: cpuSt.ThrottlingData.ThrottledTime,
},
}
memSt := lcSt.CgroupStats.MemoryStats
pbSt.CgroupStats.MemoryStats = &types.MemoryStats{
Cache: memSt.Cache,
Usage: &types.MemoryData{
Usage: memSt.Usage.Usage,
MaxUsage: memSt.Usage.MaxUsage,
Failcnt: memSt.Usage.Failcnt,
},
SwapUsage: &types.MemoryData{
Usage: memSt.SwapUsage.Usage,
MaxUsage: memSt.SwapUsage.MaxUsage,
Failcnt: memSt.SwapUsage.Failcnt,
},
}
blkSt := lcSt.CgroupStats.BlkioStats
pbSt.CgroupStats.BlkioStats = &types.BlkioStats{
IoServiceBytesRecursive: convertBlkioEntryToPb(blkSt.IoServiceBytesRecursive),
IoServicedRecursive: convertBlkioEntryToPb(blkSt.IoServicedRecursive),
IoQueuedRecursive: convertBlkioEntryToPb(blkSt.IoQueuedRecursive),
IoServiceTimeRecursive: convertBlkioEntryToPb(blkSt.IoServiceTimeRecursive),
IoWaitTimeRecursive: convertBlkioEntryToPb(blkSt.IoWaitTimeRecursive),
IoMergedRecursive: convertBlkioEntryToPb(blkSt.IoMergedRecursive),
IoTimeRecursive: convertBlkioEntryToPb(blkSt.IoTimeRecursive),
SectorsRecursive: convertBlkioEntryToPb(blkSt.SectorsRecursive),
}
pbSt.CgroupStats.HugetlbStats = make(map[string]*types.HugetlbStats)
for k, st := range lcSt.CgroupStats.HugetlbStats {
pbSt.CgroupStats.HugetlbStats[k] = &types.HugetlbStats{
Usage: st.Usage,
MaxUsage: st.MaxUsage,
Failcnt: st.Failcnt,
}
}
return pbSt
}
func convertBlkioEntryToPb(b []cgroups.BlkioStatEntry) []*types.BlkioStatsEntry {
var pbEs []*types.BlkioStatsEntry
for _, e := range b {
pbEs = append(pbEs, &types.BlkioStatsEntry{
Major: e.Major,
Minor: e.Minor,
Op: e.Op,
Value: e.Value,
})
}
return pbEs
} }

285
api/grpc/types/api.pb.go
View file

@ -45,7 +45,7 @@ It has these top-level messages:
BlkioStats BlkioStats
HugetlbStats HugetlbStats
CgroupStats CgroupStats
Stats StatsResponse
StatsRequest StatsRequest
*/ */
package types package types
@ -677,25 +677,25 @@ func (m *CgroupStats) GetHugetlbStats() map[string]*HugetlbStats {
return nil return nil
} }
type Stats struct { type StatsResponse struct {
NetworkStats []*NetworkStats `protobuf:"bytes,1,rep,name=network_stats" json:"network_stats,omitempty"` NetworkStats []*NetworkStats `protobuf:"bytes,1,rep,name=network_stats" json:"network_stats,omitempty"`
CgroupStats *CgroupStats `protobuf:"bytes,2,opt,name=cgroup_stats" json:"cgroup_stats,omitempty"` CgroupStats *CgroupStats `protobuf:"bytes,2,opt,name=cgroup_stats" json:"cgroup_stats,omitempty"`
Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp" json:"timestamp,omitempty"` Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp" json:"timestamp,omitempty"`
} }
func (m *Stats) Reset() { *m = Stats{} } func (m *StatsResponse) Reset() { *m = StatsResponse{} }
func (m *Stats) String() string { return proto.CompactTextString(m) } func (m *StatsResponse) String() string { return proto.CompactTextString(m) }
func (*Stats) ProtoMessage() {} func (*StatsResponse) ProtoMessage() {}
func (*Stats) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} } func (*StatsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} }
func (m *Stats) GetNetworkStats() []*NetworkStats { func (m *StatsResponse) GetNetworkStats() []*NetworkStats {
if m != nil { if m != nil {
return m.NetworkStats return m.NetworkStats
} }
return nil return nil
} }
func (m *Stats) GetCgroupStats() *CgroupStats { func (m *StatsResponse) GetCgroupStats() *CgroupStats {
if m != nil { if m != nil {
return m.CgroupStats return m.CgroupStats
} }
@ -748,7 +748,7 @@ func init() {
proto.RegisterType((*BlkioStats)(nil), "types.BlkioStats") proto.RegisterType((*BlkioStats)(nil), "types.BlkioStats")
proto.RegisterType((*HugetlbStats)(nil), "types.HugetlbStats") proto.RegisterType((*HugetlbStats)(nil), "types.HugetlbStats")
proto.RegisterType((*CgroupStats)(nil), "types.CgroupStats") proto.RegisterType((*CgroupStats)(nil), "types.CgroupStats")
proto.RegisterType((*Stats)(nil), "types.Stats") proto.RegisterType((*StatsResponse)(nil), "types.StatsResponse")
proto.RegisterType((*StatsRequest)(nil), "types.StatsRequest") proto.RegisterType((*StatsRequest)(nil), "types.StatsRequest")
} }
@ -769,7 +769,7 @@ type APIClient interface {
ListCheckpoint(ctx context.Context, in *ListCheckpointRequest, opts ...grpc.CallOption) (*ListCheckpointResponse, error) ListCheckpoint(ctx context.Context, in *ListCheckpointRequest, opts ...grpc.CallOption) (*ListCheckpointResponse, error)
State(ctx context.Context, in *StateRequest, opts ...grpc.CallOption) (*StateResponse, error) State(ctx context.Context, in *StateRequest, opts ...grpc.CallOption) (*StateResponse, error)
Events(ctx context.Context, in *EventsRequest, opts ...grpc.CallOption) (API_EventsClient, error) Events(ctx context.Context, in *EventsRequest, opts ...grpc.CallOption) (API_EventsClient, error)
GetStats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (API_GetStatsClient, error) Stats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (*StatsResponse, error)
} }
type aPIClient struct { type aPIClient struct {
@ -893,36 +893,13 @@ func (x *aPIEventsClient) Recv() (*Event, error) {
return m, nil return m, nil
} }
func (c *aPIClient) GetStats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (API_GetStatsClient, error) { func (c *aPIClient) Stats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (*StatsResponse, error) {
stream, err := grpc.NewClientStream(ctx, &_API_serviceDesc.Streams[1], c.cc, "/types.API/GetStats", opts...) out := new(StatsResponse)
err := grpc.Invoke(ctx, "/types.API/Stats", in, out, c.cc, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
x := &aPIGetStatsClient{stream} return out, nil
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
if err := x.ClientStream.CloseSend(); err != nil {
return nil, err
}
return x, nil
}
type API_GetStatsClient interface {
Recv() (*Stats, error)
grpc.ClientStream
}
type aPIGetStatsClient struct {
grpc.ClientStream
}
func (x *aPIGetStatsClient) Recv() (*Stats, error) {
m := new(Stats)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
} }
// Server API for API service // Server API for API service
@ -938,7 +915,7 @@ type APIServer interface {
ListCheckpoint(context.Context, *ListCheckpointRequest) (*ListCheckpointResponse, error) ListCheckpoint(context.Context, *ListCheckpointRequest) (*ListCheckpointResponse, error)
State(context.Context, *StateRequest) (*StateResponse, error) State(context.Context, *StateRequest) (*StateResponse, error)
Events(*EventsRequest, API_EventsServer) error Events(*EventsRequest, API_EventsServer) error
GetStats(*StatsRequest, API_GetStatsServer) error Stats(context.Context, *StatsRequest) (*StatsResponse, error)
} }
func RegisterAPIServer(s *grpc.Server, srv APIServer) { func RegisterAPIServer(s *grpc.Server, srv APIServer) {
@ -1074,25 +1051,16 @@ func (x *aPIEventsServer) Send(m *Event) error {
return x.ServerStream.SendMsg(m) return x.ServerStream.SendMsg(m)
} }
func _API_GetStats_Handler(srv interface{}, stream grpc.ServerStream) error { func _API_Stats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error) (interface{}, error) {
m := new(StatsRequest) in := new(StatsRequest)
if err := stream.RecvMsg(m); err != nil { if err := dec(in); err != nil {
return err return nil, err
} }
return srv.(APIServer).GetStats(m, &aPIGetStatsServer{stream}) out, err := srv.(APIServer).Stats(ctx, in)
} if err != nil {
return nil, err
type API_GetStatsServer interface { }
Send(*Stats) error return out, nil
grpc.ServerStream
}
type aPIGetStatsServer struct {
grpc.ServerStream
}
func (x *aPIGetStatsServer) Send(m *Stats) error {
return x.ServerStream.SendMsg(m)
} }
var _API_serviceDesc = grpc.ServiceDesc{ var _API_serviceDesc = grpc.ServiceDesc{
@ -1135,6 +1103,10 @@ var _API_serviceDesc = grpc.ServiceDesc{
MethodName: "State", MethodName: "State",
Handler: _API_State_Handler, Handler: _API_State_Handler,
}, },
{
MethodName: "Stats",
Handler: _API_Stats_Handler,
},
}, },
Streams: []grpc.StreamDesc{ Streams: []grpc.StreamDesc{
{ {
@ -1142,111 +1114,106 @@ var _API_serviceDesc = grpc.ServiceDesc{
Handler: _API_Events_Handler, Handler: _API_Events_Handler,
ServerStreams: true, ServerStreams: true,
}, },
{
StreamName: "GetStats",
Handler: _API_GetStats_Handler,
ServerStreams: true,
},
}, },
} }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 1544 bytes of a gzipped FileDescriptorProto // 1542 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x6e, 0xdc, 0x54, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xd9, 0x6e, 0x1c, 0x45,
0x18, 0xce, 0x8c, 0x3d, 0xdb, 0x3f, 0x4b, 0x12, 0x67, 0x9b, 0x4c, 0x29, 0x0d, 0x6e, 0xa1, 0x15, 0x17, 0xf6, 0xec, 0x33, 0x67, 0x16, 0xdb, 0xed, 0x6d, 0x3c, 0xf9, 0xf3, 0xc7, 0x74, 0x02, 0x89,
0xaa, 0xa2, 0x92, 0xb2, 0x94, 0x22, 0x01, 0x25, 0xad, 0x5a, 0x50, 0x0b, 0x51, 0x93, 0x20, 0x71, 0x50, 0x64, 0x05, 0x87, 0x25, 0x04, 0x09, 0x08, 0x4e, 0x44, 0x40, 0x09, 0x58, 0xb1, 0x8d, 0xc4,
0xc3, 0xc8, 0x63, 0x1f, 0x66, 0x0e, 0xf1, 0x86, 0x7d, 0x9c, 0xe5, 0x15, 0xca, 0x25, 0xcf, 0x81, 0x0d, 0xa3, 0x9e, 0xee, 0x62, 0xa6, 0x71, 0x6f, 0x74, 0x55, 0x7b, 0x79, 0x85, 0x70, 0xcf, 0x5b,
0xc4, 0x15, 0x0f, 0xc0, 0x13, 0xf0, 0x1c, 0x3c, 0x05, 0xff, 0x59, 0xec, 0xb1, 0x3d, 0x4b, 0xe9, 0x20, 0x71, 0xc5, 0x03, 0xf0, 0x04, 0x3c, 0x07, 0x4f, 0xc1, 0xa9, 0xa5, 0x6b, 0xba, 0x7b, 0x16,
0x05, 0x37, 0x23, 0x9d, 0x73, 0xfe, 0xe5, 0xfb, 0xbf, 0x7f, 0xf1, 0x39, 0x03, 0x2d, 0x2b, 0xa4, 0x93, 0x0b, 0x6e, 0x46, 0xaa, 0xaa, 0xb3, 0x7c, 0xe7, 0x3b, 0x4b, 0x57, 0x0d, 0xb4, 0xac, 0xc8,
0xfb, 0x61, 0x14, 0xb0, 0xc0, 0xa8, 0xb1, 0xab, 0x90, 0xc4, 0xe6, 0x08, 0x36, 0x4f, 0x43, 0xc7, 0xdd, 0x8f, 0xe2, 0x90, 0x85, 0x46, 0x8d, 0x5d, 0x45, 0x84, 0x9a, 0x23, 0xd8, 0x3c, 0x8d, 0x1c,
0x62, 0xe4, 0x28, 0x0a, 0x6c, 0x12, 0xc7, 0x2f, 0xc9, 0x2f, 0x09, 0x89, 0x99, 0x01, 0x50, 0xa5, 0x8b, 0x91, 0xa3, 0x38, 0xb4, 0x09, 0xa5, 0xaf, 0xc8, 0xcf, 0x09, 0xa1, 0xcc, 0x00, 0x28, 0xbb,
0x4e, 0xbf, 0xb2, 0x57, 0xb9, 0xd3, 0x32, 0xda, 0xa0, 0x85, 0xb8, 0xa8, 0x8a, 0x05, 0x9e, 0xd8, 0x4e, 0xbf, 0xb4, 0x57, 0xba, 0xd7, 0x32, 0xda, 0x50, 0x89, 0x70, 0x51, 0x16, 0x0b, 0x3c, 0xb1,
0x6e, 0x10, 0x93, 0x63, 0xe6, 0x50, 0xbf, 0xaf, 0xe1, 0x5e, 0xd3, 0xe8, 0x42, 0xed, 0x82, 0x3a, 0xbd, 0x90, 0x92, 0x63, 0xe6, 0xb8, 0x41, 0xbf, 0x82, 0x7b, 0x4d, 0xa3, 0x0b, 0xb5, 0x0b, 0xd7,
0x6c, 0xd2, 0xd7, 0x71, 0xd9, 0x35, 0x7a, 0x50, 0x9f, 0x10, 0x3a, 0x9e, 0xb0, 0x7e, 0x8d, 0xaf, 0x61, 0x93, 0x7e, 0x15, 0x97, 0x5d, 0xa3, 0x07, 0xf5, 0x09, 0x71, 0xc7, 0x13, 0xd6, 0xaf, 0xf1,
0xcd, 0x1d, 0xd8, 0x2a, 0xf9, 0x88, 0xc3, 0xc0, 0x8f, 0x89, 0xf9, 0x6b, 0x05, 0xb6, 0x0f, 0x23, 0xb5, 0xb9, 0x03, 0x5b, 0x05, 0x1f, 0x34, 0x0a, 0x03, 0x4a, 0xcc, 0x5f, 0x4a, 0xb0, 0x7d, 0x18,
0x82, 0x27, 0x87, 0x81, 0xcf, 0x2c, 0xea, 0x93, 0x68, 0x9e, 0x7f, 0x5c, 0x8c, 0x12, 0xdf, 0x71, 0x13, 0x3c, 0x39, 0x0c, 0x03, 0x66, 0xb9, 0x01, 0x89, 0xe7, 0xf9, 0xc7, 0xc5, 0x28, 0x09, 0x1c,
0xc9, 0x91, 0x85, 0x3e, 0xa6, 0x30, 0x26, 0xc4, 0x3e, 0x0b, 0x03, 0xea, 0x33, 0x01, 0xa3, 0xc5, 0x8f, 0x1c, 0x59, 0xe8, 0x63, 0x0a, 0x63, 0x42, 0xec, 0xb3, 0x28, 0x74, 0x03, 0x26, 0x60, 0xb4,
0x61, 0xc4, 0x02, 0x95, 0x2e, 0x96, 0x08, 0x03, 0x97, 0x41, 0x22, 0x61, 0xa4, 0x6b, 0x12, 0x45, 0x38, 0x0c, 0x2a, 0x50, 0x55, 0xc5, 0x12, 0x61, 0xe0, 0x32, 0x4c, 0x24, 0x8c, 0x74, 0x4d, 0xe2,
0xfd, 0x7a, 0xba, 0x76, 0xad, 0x11, 0x71, 0xe3, 0x7e, 0x63, 0x4f, 0xbb, 0xd3, 0x32, 0x3f, 0x87, 0xb8, 0x5f, 0x4f, 0xd7, 0x9e, 0x35, 0x22, 0x1e, 0xed, 0x37, 0xf6, 0x2a, 0xf7, 0x5a, 0xe6, 0xa7,
0x9d, 0x19, 0x30, 0x12, 0xa8, 0x71, 0x13, 0x5a, 0x76, 0xba, 0x29, 0x40, 0xb5, 0x0f, 0xd6, 0xf6, 0xb0, 0x33, 0x03, 0x46, 0x02, 0x35, 0x6e, 0x43, 0xcb, 0x4e, 0x37, 0x05, 0xa8, 0xf6, 0xc1, 0xda,
0x05, 0x81, 0xfb, 0x99, 0xb0, 0xf9, 0x00, 0xba, 0xc7, 0x74, 0xec, 0x5b, 0xee, 0x6b, 0x39, 0xe4, 0xbe, 0x20, 0x70, 0x5f, 0x0b, 0x9b, 0x8f, 0xa0, 0x7b, 0xec, 0x8e, 0x03, 0xcb, 0xbb, 0x96, 0x43,
0x48, 0x84, 0xa4, 0x00, 0xde, 0x35, 0xd7, 0xa0, 0x97, 0x6a, 0x2a, 0x66, 0xfe, 0xa8, 0xc0, 0xfa, 0x8e, 0x44, 0x48, 0x0a, 0xe0, 0x5d, 0x73, 0x0d, 0x7a, 0xa9, 0xa6, 0x62, 0xe6, 0xf7, 0x12, 0xac,
0x23, 0xc7, 0x59, 0x92, 0x94, 0x35, 0x68, 0x32, 0x12, 0x79, 0x94, 0x5b, 0xa9, 0x8a, 0x2c, 0xec, 0x3f, 0x71, 0x9c, 0x25, 0x49, 0x59, 0x83, 0x26, 0x23, 0xb1, 0xef, 0x72, 0x2b, 0x65, 0x91, 0x85,
0x82, 0x9e, 0xc4, 0x88, 0x4f, 0x13, 0xf8, 0xda, 0x0a, 0xdf, 0x29, 0x6e, 0x19, 0x1d, 0xd0, 0xad, 0x5d, 0xa8, 0x26, 0x14, 0xf1, 0x55, 0x04, 0xbe, 0xb6, 0xc2, 0x77, 0x8a, 0x5b, 0x46, 0x07, 0xaa,
0x68, 0x1c, 0x23, 0x31, 0x9a, 0xc4, 0x42, 0xfc, 0x73, 0x64, 0x45, 0x2d, 0xec, 0x0b, 0x47, 0x51, 0x56, 0x3c, 0xa6, 0x48, 0x4c, 0x45, 0x62, 0x21, 0xc1, 0x39, 0xb2, 0xa2, 0x16, 0xf6, 0x85, 0xa3,
0xa2, 0x50, 0x36, 0x8a, 0x74, 0x36, 0x4b, 0x74, 0xb6, 0x4a, 0x74, 0x02, 0x5f, 0x63, 0xf8, 0xba, 0x28, 0x51, 0x28, 0x1b, 0x79, 0x3a, 0x9b, 0x05, 0x3a, 0x5b, 0x05, 0x3a, 0x81, 0xaf, 0x31, 0xfc,
0xf0, 0x85, 0x36, 0x12, 0x85, 0xb2, 0xcb, 0x17, 0x63, 0x15, 0x76, 0xd7, 0xd8, 0x86, 0x9e, 0xe5, 0xaa, 0xf0, 0x85, 0x36, 0x12, 0x85, 0xb2, 0xcb, 0x17, 0x63, 0x15, 0x76, 0xd7, 0xd8, 0x86, 0x9e,
0x38, 0x94, 0xd1, 0x00, 0x41, 0x3f, 0xa5, 0x4e, 0x8c, 0x50, 0x35, 0x0c, 0x7f, 0x13, 0x8c, 0x7c, 0xe5, 0x38, 0x2e, 0x73, 0x43, 0x04, 0xfd, 0xa5, 0xeb, 0x50, 0x84, 0x5a, 0xc1, 0xf0, 0x37, 0xc1,
0xac, 0x8a, 0x82, 0xe7, 0x59, 0x3a, 0xb2, 0x3c, 0xcf, 0xe3, 0xe1, 0xdd, 0x42, 0x21, 0x54, 0x45, 0xc8, 0xc6, 0xaa, 0x28, 0x78, 0xa1, 0xd3, 0xa1, 0xf3, 0x3c, 0x8f, 0x87, 0xb7, 0x73, 0x85, 0x50,
0xec, 0xeb, 0x69, 0x6e, 0xb2, 0x03, 0x73, 0x00, 0xfd, 0x59, 0x6b, 0xca, 0xd3, 0x7d, 0xd8, 0x79, 0x16, 0xb1, 0xaf, 0xa7, 0xb9, 0xd1, 0x07, 0xe6, 0x00, 0xfa, 0xb3, 0xd6, 0x94, 0xa7, 0x87, 0xb0,
0x4c, 0x5c, 0xf2, 0x3a, 0x4f, 0x48, 0xa2, 0x6f, 0x79, 0x44, 0xe6, 0x90, 0x1b, 0x9c, 0x55, 0x52, 0xf3, 0x94, 0x78, 0xe4, 0x3a, 0x4f, 0x48, 0x62, 0x60, 0xf9, 0x44, 0xe6, 0x90, 0x1b, 0x9c, 0x55,
0x06, 0x6f, 0xc2, 0xd6, 0x73, 0x1a, 0xb3, 0xa5, 0xe6, 0xcc, 0x1f, 0x00, 0xa6, 0x02, 0x99, 0xf1, 0x52, 0x06, 0x6f, 0xc3, 0xd6, 0x0b, 0x97, 0xb2, 0xa5, 0xe6, 0xcc, 0xef, 0x01, 0xa6, 0x02, 0xda,
0xcc, 0x15, 0xb9, 0xa4, 0x4c, 0x25, 0x16, 0x49, 0x64, 0x76, 0xa8, 0x7a, 0x6d, 0x03, 0xda, 0x89, 0xb8, 0x76, 0x45, 0x2e, 0x5d, 0xa6, 0x12, 0x8b, 0x24, 0x32, 0x3b, 0x52, 0xbd, 0xb6, 0x01, 0xed,
0x4f, 0x2f, 0x8f, 0x03, 0xfb, 0x8c, 0xb0, 0x58, 0x94, 0xba, 0x68, 0xc0, 0x78, 0x42, 0x5c, 0x57, 0x24, 0x70, 0x2f, 0x8f, 0x43, 0xfb, 0x8c, 0x30, 0x2a, 0x4a, 0x5d, 0x34, 0x20, 0x9d, 0x10, 0xcf,
0x54, 0x7a, 0xd3, 0xfc, 0x12, 0xb6, 0xcb, 0xfe, 0x55, 0x21, 0xbf, 0x07, 0xed, 0x29, 0x5b, 0x31, 0x13, 0x95, 0xde, 0x34, 0x3f, 0x87, 0xed, 0xa2, 0x7f, 0x55, 0xc8, 0xef, 0x40, 0x7b, 0xca, 0x16,
0x7a, 0xd3, 0x16, 0xd1, 0xd5, 0x39, 0x66, 0xc8, 0xd6, 0x3c, 0xe0, 0x7b, 0xd0, 0xcb, 0x8a, 0x5e, 0x45, 0x6f, 0x95, 0x45, 0x74, 0x75, 0x8e, 0x19, 0xb2, 0x35, 0x0f, 0xf8, 0x1e, 0xf4, 0x74, 0xd1,
0x08, 0xc9, 0x52, 0xb0, 0x58, 0x12, 0x2b, 0x89, 0xdf, 0x2b, 0xd0, 0x50, 0xe9, 0x4c, 0x4b, 0xea, 0x0b, 0x21, 0x59, 0x0a, 0x16, 0x4b, 0xa8, 0x92, 0xf8, 0xad, 0x04, 0x0d, 0x95, 0xce, 0xb4, 0xa4,
0x7f, 0x2c, 0xda, 0x75, 0x68, 0xc5, 0x57, 0x31, 0x23, 0xde, 0x91, 0x2a, 0xdd, 0xee, 0x9b, 0x96, 0xfe, 0xc3, 0xa2, 0x5d, 0x87, 0x16, 0xbd, 0xa2, 0x8c, 0xf8, 0x47, 0xaa, 0x74, 0xbb, 0x6f, 0x5a,
0xee, 0xab, 0x0a, 0xb4, 0xb2, 0x90, 0x16, 0xe7, 0xbc, 0x34, 0x88, 0xe4, 0xd0, 0x79, 0x07, 0x5a, 0xba, 0xaf, 0x4b, 0xd0, 0xd2, 0x21, 0x2d, 0xce, 0x79, 0x61, 0x10, 0xc9, 0xa1, 0xf3, 0x16, 0xb4,
0xa1, 0x0c, 0x95, 0x48, 0xa8, 0xed, 0x83, 0x9e, 0x8a, 0x22, 0xa5, 0x60, 0x4a, 0x4f, 0xad, 0x34, 0x22, 0x19, 0x2a, 0x91, 0x50, 0xdb, 0x07, 0x3d, 0x15, 0x45, 0x4a, 0xc1, 0x94, 0x9e, 0x5a, 0x61,
0x78, 0xea, 0x22, 0x1a, 0x74, 0x12, 0xf2, 0x6e, 0x68, 0x88, 0x6e, 0xb8, 0x0d, 0x8d, 0x17, 0x96, 0xf0, 0xd4, 0x45, 0x34, 0xe8, 0x24, 0xe2, 0xdd, 0xd0, 0x10, 0xdd, 0x70, 0x17, 0x1a, 0x2f, 0x2d,
0x3d, 0x41, 0x28, 0xfc, 0xc0, 0x0e, 0x15, 0xab, 0x62, 0xac, 0x7a, 0xc4, 0x0b, 0xa2, 0x2b, 0x81, 0x7b, 0x82, 0x50, 0xf8, 0x81, 0x1d, 0x29, 0x56, 0xc5, 0x58, 0xf5, 0x89, 0x1f, 0xc6, 0x57, 0x02,
0x46, 0x37, 0xbf, 0xc7, 0x79, 0x23, 0x73, 0xa4, 0x92, 0x7b, 0x0b, 0x5b, 0x21, 0x8d, 0x22, 0xcd, 0x4d, 0xd5, 0xfc, 0x0e, 0xe7, 0x8d, 0xcc, 0x91, 0x4a, 0xee, 0x1d, 0x6c, 0x85, 0x34, 0x8a, 0x34,
0xed, 0xcc, 0x98, 0x32, 0x6e, 0x40, 0xc3, 0x93, 0xf6, 0x55, 0xb7, 0xa4, 0x70, 0x95, 0x57, 0xf3, 0xb7, 0x33, 0x63, 0xca, 0xb8, 0x05, 0x0d, 0x5f, 0xda, 0x57, 0xdd, 0x92, 0xc2, 0x55, 0x5e, 0xcd,
0x11, 0x6c, 0xcb, 0x71, 0xbd, 0x74, 0x28, 0xcf, 0x0c, 0x34, 0x19, 0xa1, 0x20, 0xc5, 0xdc, 0x85, 0x27, 0xb0, 0x2d, 0xc7, 0xf5, 0xd2, 0xa1, 0x3c, 0x33, 0xd0, 0x64, 0x84, 0x82, 0x14, 0x73, 0x17,
0x9d, 0x19, 0x13, 0xaa, 0x37, 0x56, 0xa1, 0xfb, 0xe4, 0x9c, 0x60, 0xf1, 0x29, 0xa3, 0xe6, 0xdf, 0x76, 0x66, 0x4c, 0xa8, 0xde, 0x58, 0x85, 0xee, 0xb3, 0x73, 0x82, 0xc5, 0xa7, 0x8c, 0x9a, 0x7f,
0x15, 0xa8, 0x89, 0x1d, 0x1e, 0x2e, 0x47, 0xa2, 0x1c, 0x48, 0x67, 0xf3, 0xec, 0x77, 0x4b, 0x89, 0x95, 0xa0, 0x26, 0x76, 0x78, 0xb8, 0x1c, 0x89, 0x72, 0x20, 0x9d, 0xcd, 0xb3, 0xdf, 0x2d, 0x24,
0xd0, 0xf3, 0x80, 0x6a, 0xa5, 0x09, 0x2b, 0x0b, 0x02, 0x83, 0x56, 0x59, 0x12, 0x25, 0x31, 0x9b, 0xa2, 0x9a, 0x05, 0x54, 0x2b, 0x4c, 0x58, 0x59, 0x10, 0x18, 0xb4, 0xca, 0x92, 0x28, 0x89, 0xd9,
0xa3, 0x22, 0x77, 0xad, 0x05, 0xdc, 0x15, 0x87, 0x0d, 0x2c, 0x1a, 0x36, 0x7f, 0x56, 0xa0, 0xf3, 0x1c, 0xe5, 0xb9, 0x6b, 0x2d, 0xe0, 0x2e, 0x3f, 0x6c, 0x60, 0xd1, 0xb0, 0xf9, 0xa3, 0x04, 0x9d,
0x2d, 0x61, 0x17, 0x41, 0x74, 0xc6, 0x33, 0x14, 0x97, 0xba, 0x1b, 0xbb, 0x20, 0xba, 0x1c, 0x8e, 0x6f, 0x08, 0xbb, 0x08, 0xe3, 0x33, 0x9e, 0x21, 0x5a, 0xe8, 0x6e, 0xec, 0x82, 0xf8, 0x72, 0x38,
0xae, 0x18, 0x56, 0x8c, 0x48, 0x25, 0x8f, 0x07, 0x77, 0x8e, 0x2c, 0xd9, 0xd3, 0x9a, 0xd8, 0xc3, 0xba, 0x62, 0x58, 0x31, 0x22, 0x95, 0x3c, 0x1e, 0xdc, 0x39, 0xb2, 0x64, 0x4f, 0x57, 0xc4, 0x1e,
0xb2, 0x7e, 0x79, 0x39, 0xc4, 0x22, 0x0d, 0x22, 0xd9, 0xe6, 0x42, 0x0c, 0xb7, 0x9c, 0x28, 0x08, 0x96, 0xf5, 0xab, 0xcb, 0x21, 0x16, 0x69, 0x18, 0xcb, 0x36, 0x17, 0x62, 0xb8, 0xe5, 0xc4, 0x61,
0x43, 0x22, 0x23, 0xd5, 0xb9, 0xb1, 0x93, 0xd4, 0x58, 0x3d, 0x95, 0xc2, 0x9d, 0x50, 0x19, 0x6b, 0x14, 0x11, 0x19, 0x69, 0x95, 0x1b, 0x3b, 0x49, 0x8d, 0xd5, 0x53, 0x29, 0xdc, 0x89, 0x94, 0xb1,
0xa4, 0xc6, 0x4e, 0x32, 0x63, 0xcd, 0x9c, 0x58, 0x6a, 0xac, 0x25, 0x4a, 0xca, 0x83, 0xe6, 0x61, 0x46, 0x6a, 0xec, 0x44, 0x1b, 0x6b, 0x66, 0xc4, 0x52, 0x63, 0x2d, 0x51, 0x52, 0x3e, 0x34, 0x0f,
0x98, 0x9c, 0xc6, 0xd6, 0x98, 0xf0, 0x41, 0xc3, 0x02, 0x66, 0xb9, 0xc3, 0x84, 0x2f, 0x05, 0x74, 0xa3, 0xe4, 0x94, 0x5a, 0x63, 0xc2, 0x07, 0x0d, 0x0b, 0x99, 0xe5, 0x0d, 0x13, 0xbe, 0x14, 0xd0,
0xdd, 0xd8, 0x84, 0x4e, 0x48, 0x22, 0x2c, 0x4a, 0xb5, 0x5b, 0x45, 0xa2, 0x74, 0xe3, 0x1a, 0x6c, 0xab, 0xc6, 0x26, 0x74, 0x22, 0x12, 0x63, 0x51, 0xaa, 0xdd, 0x32, 0x12, 0x55, 0x35, 0x6e, 0xc0,
0x88, 0xe5, 0x90, 0xfa, 0xc3, 0x33, 0x12, 0xf9, 0xc4, 0xf5, 0x02, 0x87, 0xa8, 0x38, 0x76, 0x61, 0x86, 0x58, 0x0e, 0xdd, 0x60, 0x78, 0x46, 0xe2, 0x80, 0x78, 0x7e, 0xe8, 0x10, 0x15, 0xc7, 0x2e,
0x3d, 0x3b, 0xe4, 0xad, 0x2e, 0x8e, 0x44, 0x3c, 0xe6, 0x09, 0xf4, 0x4e, 0x26, 0x78, 0x1b, 0x61, 0xac, 0xeb, 0x43, 0xde, 0xea, 0xe2, 0x48, 0xc4, 0x63, 0x9e, 0x40, 0xef, 0x64, 0x82, 0xb7, 0x11,
0x2e, 0xf5, 0xc7, 0x8f, 0x2d, 0x66, 0x19, 0xab, 0x98, 0x27, 0x12, 0xd1, 0xc0, 0x89, 0x95, 0x43, 0xe6, 0xb9, 0xc1, 0xf8, 0xa9, 0xc5, 0x2c, 0x63, 0x15, 0xf3, 0x44, 0x62, 0x37, 0x74, 0xa8, 0x72,
0xd4, 0x66, 0x52, 0x84, 0x38, 0xc3, 0xf4, 0x48, 0x92, 0x86, 0x9f, 0x93, 0xe9, 0x11, 0xa3, 0x9e, 0x88, 0xda, 0x4c, 0x8a, 0x10, 0x67, 0x98, 0x1e, 0x49, 0xd2, 0xf0, 0x73, 0x32, 0x3d, 0x62, 0xae,
0x72, 0x68, 0xfe, 0x28, 0x82, 0x90, 0xc4, 0x9b, 0xf8, 0xe1, 0xce, 0xc0, 0xca, 0x0f, 0xf7, 0x6a, 0xaf, 0x1c, 0x9a, 0x3f, 0x88, 0x20, 0x24, 0xf1, 0x26, 0x7e, 0xb8, 0x35, 0x58, 0xf9, 0xe1, 0x5e,
0x9a, 0xaf, 0x34, 0xd0, 0x7d, 0x58, 0x65, 0x19, 0x8a, 0x21, 0x56, 0xad, 0xa5, 0x1a, 0x63, 0x4b, 0x4d, 0xf3, 0x95, 0x06, 0xba, 0x0f, 0xab, 0x4c, 0xa3, 0x18, 0x62, 0xd5, 0x5a, 0xaa, 0x31, 0xb6,
0x49, 0x16, 0x31, 0x9a, 0x5f, 0x00, 0xbc, 0x10, 0x7d, 0x28, 0x10, 0xe3, 0xa8, 0xc9, 0x13, 0x84, 0x94, 0x64, 0x1e, 0xa3, 0xf9, 0x19, 0xc0, 0x4b, 0xd1, 0x87, 0x02, 0x31, 0x8e, 0x9a, 0x2c, 0x41,
0x44, 0x7b, 0xd6, 0x65, 0xc6, 0x0e, 0xdf, 0xc2, 0x98, 0x7e, 0xb2, 0xa8, 0x6b, 0xab, 0x7b, 0x8a, 0x48, 0xb4, 0x6f, 0x5d, 0x6a, 0x76, 0xf8, 0x16, 0xc6, 0xf4, 0xa3, 0xe5, 0x7a, 0xb6, 0xba, 0xa7,
0x6e, 0xfe, 0x53, 0x81, 0xb6, 0xb4, 0x20, 0x41, 0xa2, 0x09, 0x1b, 0x7b, 0x2f, 0x35, 0xb1, 0x97, 0x54, 0xcd, 0xbf, 0x4b, 0xd0, 0x96, 0x16, 0x24, 0x48, 0x34, 0x61, 0x63, 0xef, 0xa5, 0x26, 0xf6,
0x5a, 0x2c, 0x7e, 0xcc, 0x72, 0x3e, 0xb1, 0x0c, 0xe3, 0x0b, 0x2b, 0x54, 0x5e, 0xb4, 0x45, 0x62, 0x52, 0x8b, 0xf9, 0x8f, 0x59, 0xc6, 0x27, 0x96, 0x21, 0xbd, 0xb0, 0x22, 0xe5, 0xa5, 0xb2, 0x48,
0xb7, 0xa1, 0x23, 0xb3, 0xa1, 0x04, 0xf5, 0x45, 0x82, 0x77, 0xf9, 0xb8, 0x44, 0x24, 0x62, 0xba, 0xec, 0x2e, 0x74, 0x64, 0x36, 0x94, 0x60, 0x75, 0x91, 0xe0, 0x7d, 0x3e, 0x2e, 0x11, 0x89, 0x98,
0xb6, 0x0f, 0xae, 0x17, 0x24, 0x04, 0xc6, 0x7d, 0xf1, 0xfb, 0xc4, 0x67, 0xd1, 0xd5, 0xe0, 0x2e, 0xae, 0xed, 0x83, 0x9b, 0x39, 0x09, 0x81, 0x71, 0x5f, 0xfc, 0x3e, 0x0b, 0x58, 0x7c, 0x35, 0xb8,
0xc0, 0x74, 0xc5, 0xdb, 0xee, 0x8c, 0x5c, 0xa9, 0xca, 0xc6, 0x48, 0xce, 0x2d, 0x37, 0x51, 0x91, 0x0f, 0x30, 0x5d, 0xf1, 0xb6, 0x3b, 0x23, 0x57, 0xaa, 0xb2, 0x31, 0x92, 0x73, 0xcb, 0x4b, 0x54,
0x3f, 0xac, 0x3e, 0xa8, 0x98, 0xdf, 0xc0, 0xea, 0x57, 0xee, 0x19, 0x0d, 0x72, 0x2a, 0x28, 0xe5, 0xe4, 0x8f, 0xcb, 0x8f, 0x4a, 0xe6, 0xd7, 0xb0, 0xfa, 0x85, 0x77, 0xe6, 0x86, 0x19, 0x15, 0x94,
0x59, 0x3f, 0x07, 0x91, 0x8a, 0x97, 0x2f, 0xa9, 0x8f, 0x4b, 0x49, 0x17, 0xf6, 0x7d, 0x10, 0x4e, 0xf2, 0xad, 0x9f, 0xc2, 0x58, 0xc5, 0xcb, 0x97, 0x6e, 0x80, 0x4b, 0x49, 0x17, 0xf6, 0x7d, 0x18,
0x6f, 0x74, 0xd2, 0x9e, 0xac, 0x97, 0xbf, 0x34, 0x80, 0xa9, 0x31, 0xe3, 0x21, 0x0c, 0x68, 0x30, 0x4d, 0x6f, 0x74, 0xd2, 0x9e, 0xac, 0x97, 0x3f, 0x2b, 0x00, 0x53, 0x63, 0xc6, 0x63, 0x18, 0xb8,
0xc4, 0x92, 0x3a, 0xa7, 0x36, 0x91, 0x2d, 0x30, 0x8c, 0x88, 0x9d, 0x44, 0x31, 0x3d, 0x27, 0x6a, 0xe1, 0x10, 0x4b, 0xea, 0xdc, 0xb5, 0x89, 0x6c, 0x81, 0x61, 0x4c, 0xec, 0x24, 0xa6, 0xee, 0x39,
0xfe, 0x6d, 0xab, 0x58, 0xca, 0x18, 0x3e, 0x82, 0xad, 0xa9, 0xae, 0x93, 0x53, 0xab, 0x2e, 0x55, 0x51, 0xf3, 0x6f, 0x5b, 0xc5, 0x52, 0xc4, 0xf0, 0x01, 0x6c, 0x4d, 0x75, 0x9d, 0x8c, 0x5a, 0x79,
0xbb, 0x0f, 0x1b, 0xa8, 0x86, 0x83, 0x2b, 0x29, 0x28, 0x69, 0x4b, 0x95, 0x3e, 0x85, 0xdd, 0x1c, 0xa9, 0xda, 0x43, 0xd8, 0x40, 0x35, 0x1c, 0x5c, 0x49, 0x4e, 0xa9, 0xb2, 0x54, 0xe9, 0x63, 0xd8,
0x4e, 0x5e, 0xa9, 0x39, 0x55, 0x7d, 0xa9, 0xea, 0xc7, 0xb0, 0x8d, 0xaa, 0x17, 0x16, 0x65, 0x65, 0xcd, 0xe0, 0xe4, 0x95, 0x9a, 0x51, 0xad, 0x2e, 0x55, 0xfd, 0x10, 0xb6, 0x51, 0xf5, 0xc2, 0x72,
0xbd, 0xda, 0x7f, 0xc0, 0xe9, 0x91, 0x68, 0x5c, 0xc0, 0x59, 0x5f, 0xaa, 0xf4, 0x01, 0xac, 0xa3, 0x59, 0x51, 0xaf, 0xf6, 0x2f, 0x70, 0xfa, 0x24, 0x1e, 0xe7, 0x70, 0xd6, 0x97, 0x2a, 0xbd, 0x07,
0x52, 0xc9, 0x4f, 0xe3, 0x75, 0x2a, 0x31, 0xb1, 0x19, 0x4e, 0x95, 0x9c, 0x4a, 0x73, 0x99, 0x0a, 0xeb, 0xa8, 0x54, 0xf0, 0xd3, 0xb8, 0x4e, 0x85, 0x12, 0x9b, 0xe1, 0x54, 0xc9, 0xa8, 0x34, 0x97,
0x7e, 0x5e, 0x3a, 0xcf, 0x92, 0x31, 0x61, 0xee, 0x28, 0xab, 0xfe, 0x37, 0x6d, 0xa0, 0x57, 0x55, 0xa9, 0xe0, 0xe7, 0xa5, 0xf3, 0x3c, 0x19, 0x13, 0xe6, 0x8d, 0x74, 0xf5, 0xbf, 0x69, 0x03, 0xbd,
0x68, 0x1f, 0x8e, 0xa3, 0x20, 0x09, 0x0b, 0x5d, 0x2e, 0x6b, 0x78, 0xa6, 0xcb, 0xa5, 0xcc, 0x1d, 0x2e, 0x43, 0xfb, 0x70, 0x1c, 0x87, 0x49, 0x94, 0xeb, 0x72, 0x59, 0xc3, 0x33, 0x5d, 0x2e, 0x65,
0xe8, 0xc8, 0xaf, 0xa7, 0x12, 0x93, 0xcd, 0x65, 0xcc, 0x96, 0x3a, 0xbf, 0x23, 0x8d, 0x38, 0x66, 0xee, 0x41, 0x47, 0x7e, 0x3d, 0x95, 0x98, 0x6c, 0x2e, 0x63, 0xb6, 0xd4, 0xf9, 0x1d, 0x69, 0xc4,
0x25, 0x58, 0x6c, 0xaf, 0x5c, 0xf9, 0x7d, 0x06, 0xdd, 0x89, 0x0c, 0x44, 0x49, 0xca, 0x54, 0xde, 0x31, 0x2b, 0xc1, 0x7c, 0x7b, 0x65, 0xca, 0xef, 0x13, 0xe8, 0x4e, 0x64, 0x20, 0x4a, 0x52, 0xa6,
0x4a, 0x3d, 0x4f, 0x01, 0xee, 0xe7, 0x03, 0x96, 0x4d, 0xf4, 0x0c, 0xd6, 0x67, 0x36, 0x8b, 0xbd, 0xf2, 0x4e, 0xea, 0x79, 0x0a, 0x70, 0x3f, 0x1b, 0xb0, 0x6c, 0xa2, 0xe7, 0xb0, 0x3e, 0xb3, 0x99,
0x64, 0xe6, 0x7b, 0xa9, 0x7d, 0xb0, 0xa1, 0xcc, 0xe6, 0xb5, 0x44, 0x83, 0x85, 0x50, 0x93, 0x78, 0xef, 0x25, 0x33, 0xdb, 0x4b, 0xed, 0x83, 0x0d, 0x65, 0x36, 0xab, 0x25, 0x1a, 0xec, 0x52, 0x5e,
0xde, 0x87, 0xae, 0x2f, 0x3f, 0x3a, 0x19, 0x13, 0x5a, 0x4e, 0xb1, 0xf0, 0x41, 0x42, 0x36, 0x6c, 0x03, 0xf4, 0xc5, 0xd9, 0x78, 0x17, 0xba, 0x81, 0xfc, 0xf8, 0x68, 0x46, 0x2a, 0x19, 0x03, 0xb9,
0x81, 0x6f, 0x2e, 0x1b, 0x79, 0x6e, 0x31, 0x1f, 0xbc, 0x22, 0x50, 0xcc, 0x0b, 0x15, 0xfd, 0xea, 0x0f, 0x13, 0xb2, 0x62, 0x0b, 0x9c, 0x73, 0x59, 0xc9, 0x72, 0x8c, 0x79, 0xe1, 0x95, 0x81, 0x62,
0x72, 0x38, 0xef, 0x59, 0x72, 0xf0, 0x5b, 0x1d, 0xb4, 0x47, 0x47, 0x5f, 0x1b, 0x2f, 0x61, 0xb5, 0x7e, 0xa4, 0xd2, 0xa0, 0x2e, 0x89, 0xf3, 0x9e, 0x27, 0x07, 0xbf, 0xd6, 0xa1, 0xf2, 0xe4, 0xe8,
0xf4, 0x98, 0x32, 0xd2, 0xb1, 0x32, 0xff, 0xc5, 0x37, 0x78, 0x7b, 0xd1, 0xb1, 0xba, 0x38, 0xac, 0x2b, 0xe3, 0x15, 0xac, 0x16, 0x1e, 0x55, 0x46, 0x3a, 0x5e, 0xe6, 0xbf, 0xfc, 0x06, 0xff, 0x5f,
0x70, 0x9b, 0xa5, 0x5b, 0x45, 0x66, 0x73, 0xfe, 0x85, 0x25, 0xb3, 0xb9, 0xe8, 0x32, 0xb2, 0x62, 0x74, 0xac, 0x2e, 0x10, 0x2b, 0xdc, 0x66, 0xe1, 0x76, 0xa1, 0x6d, 0xce, 0xbf, 0xb8, 0x68, 0x9b,
0x7c, 0x02, 0x75, 0xf9, 0xf4, 0x32, 0x36, 0x95, 0x6c, 0xe1, 0x0d, 0x37, 0xd8, 0x2a, 0xed, 0x66, 0x8b, 0x2e, 0x25, 0x2b, 0xc6, 0x47, 0x50, 0x97, 0x4f, 0x30, 0x63, 0x53, 0xc9, 0xe6, 0xde, 0x72,
0x8a, 0xcf, 0xa1, 0x5b, 0x78, 0xd4, 0x1a, 0xd7, 0x0a, 0xbe, 0x8a, 0x2f, 0xb7, 0xc1, 0x5b, 0xf3, 0x83, 0xad, 0xc2, 0xae, 0x56, 0x7c, 0x01, 0xdd, 0xdc, 0xe3, 0xd6, 0xb8, 0x91, 0xf3, 0x95, 0x7f,
0x0f, 0x33, 0x6b, 0x87, 0x00, 0xd3, 0x27, 0x90, 0xd1, 0x57, 0xd2, 0x33, 0x2f, 0xc0, 0xc1, 0xee, 0xc1, 0x0d, 0xfe, 0x37, 0xff, 0x50, 0x5b, 0x3b, 0x04, 0x98, 0x3e, 0x85, 0x8c, 0xbe, 0x92, 0x9e,
0x9c, 0x93, 0xcc, 0xc8, 0x29, 0xac, 0x95, 0xdf, 0x38, 0x46, 0x89, 0xd5, 0xf2, 0x8b, 0x64, 0x70, 0x79, 0x09, 0x0e, 0x76, 0xe7, 0x9c, 0x68, 0x23, 0xa7, 0xb0, 0x56, 0x7c, 0xeb, 0x18, 0x05, 0x56,
0x63, 0xe1, 0x79, 0xde, 0x6c, 0xf9, 0xa5, 0x93, 0x99, 0x5d, 0xf0, 0x6e, 0xca, 0xcc, 0x2e, 0x7c, 0x8b, 0x2f, 0x93, 0xc1, 0xad, 0x85, 0xe7, 0x59, 0xb3, 0xc5, 0x17, 0x8f, 0x36, 0xbb, 0xe0, 0xfd,
0x22, 0xad, 0x18, 0xdf, 0x41, 0xaf, 0xf8, 0x48, 0x31, 0x52, 0x92, 0xe6, 0xbe, 0x9d, 0x06, 0xd7, 0xa4, 0xcd, 0x2e, 0x7c, 0x2a, 0xad, 0x18, 0xdf, 0x42, 0x2f, 0xff, 0x58, 0x31, 0x52, 0x92, 0xe6,
0x17, 0x9c, 0x66, 0x06, 0x3f, 0x94, 0x8d, 0x80, 0x37, 0x97, 0x34, 0x67, 0xb9, 0x17, 0xcc, 0x60, 0xbe, 0xa1, 0x06, 0x37, 0x17, 0x9c, 0x6a, 0x83, 0xef, 0x43, 0x4d, 0x3e, 0x4b, 0xd2, 0x8a, 0xcf,
0xb3, 0xb8, 0x99, 0x69, 0xdd, 0x83, 0xba, 0xbc, 0x8f, 0x66, 0x05, 0x50, 0xb8, 0x9e, 0x0e, 0x3a, 0xbe, 0x64, 0x06, 0x9b, 0xf9, 0x4d, 0xad, 0xf5, 0x00, 0xea, 0xf2, 0x5e, 0xaa, 0x0b, 0x20, 0x77,
0xf9, 0x5d, 0x73, 0xe5, 0x5e, 0x05, 0x67, 0x5e, 0xf3, 0x29, 0x61, 0xb2, 0x3b, 0xf2, 0xae, 0x66, 0x4d, 0x1d, 0x74, 0xb2, 0xbb, 0xe6, 0xca, 0x83, 0x52, 0xea, 0x87, 0xe6, 0xfc, 0xd0, 0x79, 0x7e,
0x54, 0xc4, 0x26, 0x57, 0x19, 0xd5, 0xc5, 0x7f, 0x2e, 0xf7, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0x32, 0xc9, 0x19, 0xd5, 0xc5, 0xdf, 0x2f, 0x0f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x7e, 0x37,
0x66, 0x65, 0xcd, 0xa0, 0x80, 0x11, 0x00, 0x00, 0x3f, 0x0d, 0x8b, 0x11, 0x00, 0x00,
} }

4
api/grpc/types/api.proto
View file

@ -13,7 +13,7 @@ service API {
rpc ListCheckpoint(ListCheckpointRequest) returns (ListCheckpointResponse) {} rpc ListCheckpoint(ListCheckpointRequest) returns (ListCheckpointResponse) {}
rpc State(StateRequest) returns (StateResponse) {} rpc State(StateRequest) returns (StateResponse) {}
rpc Events(EventsRequest) returns (stream Event) {} rpc Events(EventsRequest) returns (stream Event) {}
rpc GetStats(StatsRequest) returns (stream Stats) {} rpc Stats(StatsRequest) returns (StatsResponse) {}
} }
message UpdateProcessRequest { message UpdateProcessRequest {
@ -246,7 +246,7 @@ message CgroupStats {
map<string, HugetlbStats> hugetlb_stats = 4; // the map is in the format "size of hugepage: stats of the hugepage" map<string, HugetlbStats> hugetlb_stats = 4; // the map is in the format "size of hugepage: stats of the hugepage"
} }
message Stats { message StatsResponse {
repeated NetworkStats network_stats = 1; repeated NetworkStats network_stats = 1;
CgroupStats cgroup_stats = 2; CgroupStats cgroup_stats = 2;
uint64 timestamp = 3; uint64 timestamp = 3;

12
ctr/container.go
View file

@ -423,17 +423,15 @@ var statsCommand = cli.Command{
Id: context.Args().First(), Id: context.Args().First(),
} }
c := getClient(context) c := getClient(context)
stream, err := c.GetStats(netcontext.Background(), req) stats, err := c.Stats(netcontext.Background(), req)
if err != nil { if err != nil {
fatal(err.Error(), 1) fatal(err.Error(), 1)
} }
for { data, err := json.Marshal(stats)
stats, err := stream.Recv() if err != nil {
if err != nil { fatal(err.Error(), 1)
fatal(err.Error(), 1)
}
fmt.Println(stats)
} }
fmt.Print(string(data))
}, },
} }

33
runtime/container.go
View file

@ -46,7 +46,8 @@ type Container interface {
// Pids returns all pids inside the container // Pids returns all pids inside the container
Pids() ([]int, error) Pids() ([]int, error)
// Stats returns realtime container stats and resource information // Stats returns realtime container stats and resource information
// Stats() (*Stat, error) // OOM signals the channel if the container received an OOM notification Stats() (*Stat, error)
// OOM signals the channel if the container received an OOM notification
// OOM() (<-chan struct{}, error) // OOM() (<-chan struct{}, error)
} }
@ -357,17 +358,37 @@ func (c *container) DeleteCheckpoint(name string) error {
} }
func (c *container) Pids() ([]int, error) { func (c *container) Pids() ([]int, error) {
f, err := libcontainer.New(specs.LinuxStateDirectory, libcontainer.Cgroupfs) container, err := c.getLibctContainer()
if err != nil {
return nil, err
}
container, err := f.Load(c.id)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return container.Processes() return container.Processes()
} }
func (c *container) Stats() (*Stat, error) {
container, err := c.getLibctContainer()
if err != nil {
return nil, err
}
now := time.Now()
stats, err := container.Stats()
if err != nil {
return nil, err
}
return &Stat{
Timestamp: now,
Data: stats,
}, nil
}
func (c *container) getLibctContainer() (libcontainer.Container, error) {
f, err := libcontainer.New(specs.LinuxStateDirectory, libcontainer.Cgroupfs)
if err != nil {
return nil, err
}
return f.Load(c.id)
}
func getRootIDs(s *specs.LinuxSpec) (int, int, error) { func getRootIDs(s *specs.LinuxSpec) (int, int, error) {
if s == nil { if s == nil {
return 0, 0, nil return 0, 0, nil

4
supervisor/event.go
View file

@ -23,8 +23,6 @@ const (
CreateCheckpointEventType EventType = "createCheckpoint" CreateCheckpointEventType EventType = "createCheckpoint"
DeleteCheckpointEventType EventType = "deleteCheckpoint" DeleteCheckpointEventType EventType = "deleteCheckpoint"
StatsEventType EventType = "events" StatsEventType EventType = "events"
UnsubscribeStatsEventType EventType = "unsubscribeStats"
StopStatsEventType EventType = "stopStats"
OOMEventType EventType = "oom" OOMEventType EventType = "oom"
) )
@ -59,7 +57,7 @@ type Event struct {
Checkpoint *runtime.Checkpoint Checkpoint *runtime.Checkpoint
Err chan error Err chan error
StartResponse chan StartResponse StartResponse chan StartResponse
Stats chan interface{} Stat chan *runtime.Stat
CloseStdin bool CloseStdin bool
ResizeTty bool ResizeTty bool
Width int Width int

4
supervisor/exit.go
View file

@ -39,10 +39,6 @@ func (h *ExitEvent) Handle(e *Event) error {
ne.Pid = proc.ID() ne.Pid = proc.ID()
h.s.SendEvent(ne) h.s.SendEvent(ne)
// remove stats collection for container
stopCollect := NewEvent(StopStatsEventType)
stopCollect.ID = container.ID()
h.s.SendEvent(stopCollect)
ExitProcessTimer.UpdateSince(start) ExitProcessTimer.UpdateSince(start)
return nil return nil

2
supervisor/metrics.go
View file

@ -6,6 +6,7 @@ var (
ContainerCreateTimer = metrics.NewTimer() ContainerCreateTimer = metrics.NewTimer()
ContainerDeleteTimer = metrics.NewTimer() ContainerDeleteTimer = metrics.NewTimer()
ContainerStartTimer = metrics.NewTimer() ContainerStartTimer = metrics.NewTimer()
ContainerStatsTimer = metrics.NewTimer()
ContainersCounter = metrics.NewCounter() ContainersCounter = metrics.NewCounter()
EventSubscriberCounter = metrics.NewCounter() EventSubscriberCounter = metrics.NewCounter()
EventsCounter = metrics.NewCounter() EventsCounter = metrics.NewCounter()
@ -19,6 +20,7 @@ func Metrics() map[string]interface{} {
"container-create-time": ContainerCreateTimer, "container-create-time": ContainerCreateTimer,
"container-delete-time": ContainerDeleteTimer, "container-delete-time": ContainerDeleteTimer,
"container-start-time": ContainerStartTimer, "container-start-time": ContainerStartTimer,
"container-stats-time": ContainerStatsTimer,
"containers": ContainersCounter, "containers": ContainersCounter,
"event-subscribers": EventSubscriberCounter, "event-subscribers": EventSubscriberCounter,
"events": EventsCounter, "events": EventsCounter,

43
supervisor/stats.go
View file

@ -1,40 +1,27 @@
package supervisor package supervisor
import "time"
type StatsEvent struct { type StatsEvent struct {
s *Supervisor s *Supervisor
} }
type UnsubscribeStatsEvent struct {
s *Supervisor
}
type StopStatsEvent struct {
s *Supervisor
}
func (h *StatsEvent) Handle(e *Event) error { func (h *StatsEvent) Handle(e *Event) error {
start := time.Now()
i, ok := h.s.containers[e.ID] i, ok := h.s.containers[e.ID]
if !ok { if !ok {
return ErrContainerNotFound return ErrContainerNotFound
} }
e.Stats = h.s.statsCollector.collect(i.container) // TODO: use workers for this
return nil go func() {
} s, err := i.container.Stats()
if err != nil {
func (h *UnsubscribeStatsEvent) Handle(e *Event) error { e.Err <- err
i, ok := h.s.containers[e.ID] return
if !ok { }
return ErrContainerNotFound e.Err <- nil
} e.Stat <- s
h.s.statsCollector.unsubscribe(i.container, e.Stats) ContainerStatsTimer.UpdateSince(start)
return nil }()
} return errDeferedResponse
func (h *StopStatsEvent) Handle(e *Event) error {
i, ok := h.s.containers[e.ID]
if !ok {
return ErrContainerNotFound
}
h.s.statsCollector.stopCollection(i.container)
return nil
} }

241
supervisor/stats_collector.go
View file

@ -1,241 +0,0 @@
package supervisor
import (
"bufio"
"fmt"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/docker/containerd/api/grpc/types"
"github.com/docker/containerd/runtime"
"github.com/docker/docker/pkg/pubsub"
"github.com/opencontainers/runc/libcontainer"
"github.com/opencontainers/runc/libcontainer/cgroups"
"github.com/opencontainers/runc/libcontainer/system"
)
func convertBlkioEntryToPb(b []cgroups.BlkioStatEntry) []*types.BlkioStatsEntry {
var pbEs []*types.BlkioStatsEntry
for _, e := range b {
pbEs = append(pbEs, &types.BlkioStatsEntry{
Major: e.Major,
Minor: e.Minor,
Op: e.Op,
Value: e.Value,
})
}
return pbEs
}
func convertToPb(st *runtime.Stat) *types.Stats {
pbSt := &types.Stats{
Timestamp: uint64(st.Timestamp.Unix()),
CgroupStats: &types.CgroupStats{},
}
lcSt, ok := st.Data.(*libcontainer.Stats)
if !ok {
return pbSt
}
cpuSt := lcSt.CgroupStats.CpuStats
pbSt.CgroupStats.CpuStats = &types.CpuStats{
CpuUsage: &types.CpuUsage{
TotalUsage: cpuSt.CpuUsage.TotalUsage,
PercpuUsage: cpuSt.CpuUsage.PercpuUsage,
UsageInKernelmode: cpuSt.CpuUsage.UsageInKernelmode,
UsageInUsermode: cpuSt.CpuUsage.UsageInUsermode,
},
ThrottlingData: &types.ThrottlingData{
Periods: cpuSt.ThrottlingData.Periods,
ThrottledPeriods: cpuSt.ThrottlingData.ThrottledPeriods,
ThrottledTime: cpuSt.ThrottlingData.ThrottledTime,
},
}
memSt := lcSt.CgroupStats.MemoryStats
pbSt.CgroupStats.MemoryStats = &types.MemoryStats{
Cache: memSt.Cache,
Usage: &types.MemoryData{
Usage: memSt.Usage.Usage,
MaxUsage: memSt.Usage.MaxUsage,
Failcnt: memSt.Usage.Failcnt,
},
SwapUsage: &types.MemoryData{
Usage: memSt.SwapUsage.Usage,
MaxUsage: memSt.SwapUsage.MaxUsage,
Failcnt: memSt.SwapUsage.Failcnt,
},
}
blkSt := lcSt.CgroupStats.BlkioStats
pbSt.CgroupStats.BlkioStats = &types.BlkioStats{
IoServiceBytesRecursive: convertBlkioEntryToPb(blkSt.IoServiceBytesRecursive),
IoServicedRecursive: convertBlkioEntryToPb(blkSt.IoServicedRecursive),
IoQueuedRecursive: convertBlkioEntryToPb(blkSt.IoQueuedRecursive),
IoServiceTimeRecursive: convertBlkioEntryToPb(blkSt.IoServiceTimeRecursive),
IoWaitTimeRecursive: convertBlkioEntryToPb(blkSt.IoWaitTimeRecursive),
IoMergedRecursive: convertBlkioEntryToPb(blkSt.IoMergedRecursive),
IoTimeRecursive: convertBlkioEntryToPb(blkSt.IoTimeRecursive),
SectorsRecursive: convertBlkioEntryToPb(blkSt.SectorsRecursive),
}
pbSt.CgroupStats.HugetlbStats = make(map[string]*types.HugetlbStats)
for k, st := range lcSt.CgroupStats.HugetlbStats {
pbSt.CgroupStats.HugetlbStats[k] = &types.HugetlbStats{
Usage: st.Usage,
MaxUsage: st.MaxUsage,
Failcnt: st.Failcnt,
}
}
return pbSt
}
type statsPair struct {
ct runtime.Container
pub *pubsub.Publisher
}
func newStatsCollector(interval time.Duration) *statsCollector {
s := &statsCollector{
interval: interval,
clockTicksPerSecond: uint64(system.GetClockTicks()),
bufReader: bufio.NewReaderSize(nil, 128),
publishers: make(map[string]*statsPair),
}
go s.run()
return s
}
// statsCollector manages and provides container resource stats
type statsCollector struct {
m sync.Mutex
supervisor *Supervisor
interval time.Duration
clockTicksPerSecond uint64
publishers map[string]*statsPair
bufReader *bufio.Reader
}
// collect registers the container with the collector and adds it to
// the event loop for collection on the specified interval returning
// a channel for the subscriber to receive on.
func (s *statsCollector) collect(c runtime.Container) chan interface{} {
s.m.Lock()
defer s.m.Unlock()
publisher, exists := s.publishers[c.ID()]
if !exists {
pub := pubsub.NewPublisher(100*time.Millisecond, 1024)
publisher = &statsPair{ct: c, pub: pub}
s.publishers[c.ID()] = publisher
}
return publisher.pub.Subscribe()
}
// stopCollection closes the channels for all subscribers and removes
// the container from metrics collection.
func (s *statsCollector) stopCollection(c runtime.Container) {
s.m.Lock()
if publisher, exists := s.publishers[c.ID()]; exists {
publisher.pub.Close()
delete(s.publishers, c.ID())
}
s.m.Unlock()
}
// unsubscribe removes a specific subscriber from receiving updates for a container's stats.
func (s *statsCollector) unsubscribe(c runtime.Container, ch chan interface{}) {
s.m.Lock()
publisher := s.publishers[c.ID()]
if publisher != nil {
publisher.pub.Evict(ch)
if publisher.pub.Len() == 0 {
delete(s.publishers, c.ID())
}
}
s.m.Unlock()
}
func (s *statsCollector) run() {
type publishersPair struct {
container runtime.Container
publisher *pubsub.Publisher
}
// we cannot determine the capacity here.
// it will grow enough in first iteration
var pairs []*statsPair
for range time.Tick(s.interval) {
// it does not make sense in the first iteration,
// but saves allocations in further iterations
pairs = pairs[:0]
s.m.Lock()
for _, publisher := range s.publishers {
// copy pointers here to release the lock ASAP
pairs = append(pairs, publisher)
}
s.m.Unlock()
if len(pairs) == 0 {
continue
}
/*
for _, pair := range pairs {
stats, err := pair.ct.Stats()
if err != nil {
logrus.Errorf("Error getting stats for container ID %s", pair.ct.ID())
continue
}
pair.pub.Publish(convertToPb(stats))
}
*/
}
}
const nanoSecondsPerSecond = 1e9
// getSystemCPUUsage returns the host system's cpu usage in
// nanoseconds. An error is returned if the format of the underlying
// file does not match.
//
// Uses /proc/stat defined by POSIX. Looks for the cpu
// statistics line and then sums up the first seven fields
// provided. See `man 5 proc` for details on specific field
// information.
func (s *statsCollector) getSystemCPUUsage() (uint64, error) {
var line string
f, err := os.Open("/proc/stat")
if err != nil {
return 0, err
}
defer func() {
s.bufReader.Reset(nil)
f.Close()
}()
s.bufReader.Reset(f)
err = nil
for err == nil {
line, err = s.bufReader.ReadString('\n')
if err != nil {
break
}
parts := strings.Fields(line)
switch parts[0] {
case "cpu":
if len(parts) < 8 {
return 0, fmt.Errorf("bad format of cpu stats")
}
var totalClockTicks uint64
for _, i := range parts[1:8] {
v, err := strconv.ParseUint(i, 10, 64)
if err != nil {
return 0, fmt.Errorf("error parsing cpu stats")
}
totalClockTicks += v
}
return (totalClockTicks * nanoSecondsPerSecond) /
s.clockTicksPerSecond, nil
}
}
return 0, fmt.Errorf("bad stats format")
}

20
supervisor/supervisor.go
View file

@ -5,7 +5,6 @@ import (
"os" "os"
"sort" "sort"
"sync" "sync"
"time"
"github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus"
"github.com/docker/containerd/chanotify" "github.com/docker/containerd/chanotify"
@ -14,7 +13,6 @@ import (
) )
const ( const (
statsInterval = 1 * time.Second
defaultBufferSize = 2048 // size of queue in eventloop defaultBufferSize = 2048 // size of queue in eventloop
) )
@ -32,14 +30,13 @@ func New(stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error)
return nil, err return nil, err
} }
s := &Supervisor{ s := &Supervisor{
stateDir: stateDir, stateDir: stateDir,
containers: make(map[string]*containerInfo), containers: make(map[string]*containerInfo),
tasks: tasks, tasks: tasks,
machine: machine, machine: machine,
subscribers: make(map[chan *Event]struct{}), subscribers: make(map[chan *Event]struct{}),
statsCollector: newStatsCollector(statsInterval), el: eventloop.NewChanLoop(defaultBufferSize),
el: eventloop.NewChanLoop(defaultBufferSize), monitor: monitor,
monitor: monitor,
} }
if oom { if oom {
s.notifier = chanotify.New() s.notifier = chanotify.New()
@ -64,8 +61,6 @@ func New(stateDir string, tasks chan *StartTask, oom bool) (*Supervisor, error)
CreateCheckpointEventType: &CreateCheckpointEvent{s}, CreateCheckpointEventType: &CreateCheckpointEvent{s},
DeleteCheckpointEventType: &DeleteCheckpointEvent{s}, DeleteCheckpointEventType: &DeleteCheckpointEvent{s},
StatsEventType: &StatsEvent{s}, StatsEventType: &StatsEvent{s},
UnsubscribeStatsEventType: &UnsubscribeStatsEvent{s},
StopStatsEventType: &StopStatsEvent{s},
UpdateProcessEventType: &UpdateProcessEvent{s}, UpdateProcessEventType: &UpdateProcessEvent{s},
} }
go s.exitHandler() go s.exitHandler()
@ -91,7 +86,6 @@ type Supervisor struct {
subscriberLock sync.RWMutex subscriberLock sync.RWMutex
subscribers map[chan *Event]struct{} subscribers map[chan *Event]struct{}
machine Machine machine Machine
statsCollector *statsCollector
notifier *chanotify.Notifier notifier *chanotify.Notifier
el eventloop.EventLoop el eventloop.EventLoop
monitor *Monitor monitor *Monitor