From 14c1c70407f58c989e6dea9b365d56cb8ee77454 Mon Sep 17 00:00:00 2001 From: Yann Ramin Date: Sun, 7 Jan 2018 14:16:46 -0800 Subject: [PATCH 1/4] Add methods for listing and fetching container stats This uses the previously unusued lib/stats.go code to return data about container stats to the CRI API. Helpers have been built around filtering based on the OCI API, and CPU stat reporting has been fixed. No data on filesystem layer usage is returned at this time. Fixes one-half of #1248 Signed-off-by: Yann Ramin --- lib/stats.go | 14 ++++--- server/container_list.go | 73 +++++++++++++++++++--------------- server/container_stats.go | 35 +++++++++++++++- server/container_stats_list.go | 37 ++++++++++++++++- 4 files changed, 118 insertions(+), 41 deletions(-) diff --git a/lib/stats.go b/lib/stats.go index 229d8409..58fac59d 100644 --- a/lib/stats.go +++ b/lib/stats.go @@ -15,8 +15,8 @@ import ( type ContainerStats struct { Container string CPU float64 - cpuNano uint64 - systemNano uint64 + CPUNano uint64 + SystemNano int64 MemUsage uint64 MemLimit uint64 MemPerc float64 @@ -29,8 +29,8 @@ type ContainerStats struct { // GetContainerStats gets the running stats for a given container func (c *ContainerServer) GetContainerStats(ctr *oci.Container, previousStats *ContainerStats) (*ContainerStats, error) { - previousCPU := previousStats.cpuNano - previousSystem := previousStats.systemNano + previousCPU := previousStats.CPUNano + previousSystem := previousStats.SystemNano libcontainerStats, err := c.LibcontainerStats(ctr) if err != nil { return nil, err @@ -38,6 +38,8 @@ func (c *ContainerServer) GetContainerStats(ctr *oci.Container, previousStats *C cgroupStats := libcontainerStats.CgroupStats stats := new(ContainerStats) stats.Container = ctr.ID() + stats.CPUNano = cgroupStats.CpuStats.CpuUsage.TotalUsage + stats.SystemNano = time.Now().UnixNano() stats.CPU = calculateCPUPercent(libcontainerStats, previousCPU, previousSystem) stats.MemUsage = cgroupStats.MemoryStats.Usage.Usage stats.MemLimit = getMemLimit(cgroupStats.MemoryStats.Usage.Limit) @@ -84,11 +86,11 @@ func getContainerNetIO(stats *libcontainer.Stats) (received uint64, transmitted return } -func calculateCPUPercent(stats *libcontainer.Stats, previousCPU, previousSystem uint64) float64 { +func calculateCPUPercent(stats *libcontainer.Stats, previousCPU uint64, previousSystem int64) float64 { var ( cpuPercent = 0.0 cpuDelta = float64(stats.CgroupStats.CpuStats.CpuUsage.TotalUsage - previousCPU) - systemDelta = float64(uint64(time.Now().UnixNano()) - previousSystem) + systemDelta = float64(uint64(time.Now().UnixNano()) - uint64(previousSystem)) ) if systemDelta > 0.0 && cpuDelta > 0.0 { // gets a ratio of container cpu usage total, multiplies it by the number of cores (4 cores running diff --git a/server/container_list.go b/server/container_list.go index 060fa2af..42ddb4e2 100644 --- a/server/container_list.go +++ b/server/container_list.go @@ -28,6 +28,44 @@ func filterContainer(c *pb.Container, filter *pb.ContainerFilter) bool { return true } +// filterContainerList applies a protobuf-defined filter to retrieve only intended containers. Not matching +// the filter is not considered an error but will return an empty response. +func (s *Server) filterContainerList(filter *pb.ContainerFilter) (ctrList []*oci.Container, err error) { + // Filter using container id and pod id first. + if filter.Id != "" { + id, err := s.CtrIDIndex().Get(filter.Id) + if err != nil { + // If we don't find a container ID with a filter, it should not + // be considered an error. Log a warning and return an empty struct + logrus.Warn("unable to find container ID %s", filter.Id) + return []*oci.Container{}, nil + } + c := s.ContainerServer.GetContainer(id) + if c != nil { + if filter.PodSandboxId != "" { + if c.Sandbox() == filter.PodSandboxId { + ctrList = []*oci.Container{c} + } else { + ctrList = []*oci.Container{} + } + + } else { + ctrList = []*oci.Container{c} + } + } + } else { + if filter.PodSandboxId != "" { + pod := s.ContainerServer.GetSandbox(filter.PodSandboxId) + if pod == nil { + ctrList = []*oci.Container{} + } else { + ctrList = pod.Containers().List() + } + } + } + return +} + // ListContainers lists all containers by filters. func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersRequest) (resp *pb.ListContainersResponse, err error) { const operation = "list_containers" @@ -45,38 +83,9 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque } if filter != nil { - - // Filter using container id and pod id first. - if filter.Id != "" { - id, err := s.CtrIDIndex().Get(filter.Id) - if err != nil { - // If we don't find a container ID with a filter, it should not - // be considered an error. Log a warning and return an empty struct - logrus.Warn("unable to find container ID %s", filter.Id) - return &pb.ListContainersResponse{}, nil - } - c := s.ContainerServer.GetContainer(id) - if c != nil { - if filter.PodSandboxId != "" { - if c.Sandbox() == filter.PodSandboxId { - ctrList = []*oci.Container{c} - } else { - ctrList = []*oci.Container{} - } - - } else { - ctrList = []*oci.Container{c} - } - } - } else { - if filter.PodSandboxId != "" { - pod := s.ContainerServer.GetSandbox(filter.PodSandboxId) - if pod == nil { - ctrList = []*oci.Container{} - } else { - ctrList = pod.Containers().List() - } - } + ctrList, err = s.filterContainerList(filter) + if err != nil { + return nil, err } } diff --git a/server/container_stats.go b/server/container_stats.go index 17df31ad..1a1558c9 100644 --- a/server/container_stats.go +++ b/server/container_stats.go @@ -4,10 +4,32 @@ import ( "fmt" "time" + "github.com/kubernetes-incubator/cri-o/lib" + "github.com/kubernetes-incubator/cri-o/oci" "golang.org/x/net/context" pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" ) +func buildContainerStats(stats *lib.ContainerStats, container *oci.Container) *pb.ContainerStats { + return &pb.ContainerStats{ + Attributes: &pb.ContainerAttributes{ + Id: container.ID(), + Metadata: container.Metadata(), + Labels: container.Labels(), + Annotations: container.Annotations(), + }, + Cpu: &pb.CpuUsage{ + Timestamp: stats.SystemNano, + UsageCoreNanoSeconds: &pb.UInt64Value{Value: stats.CPUNano}, + }, + Memory: &pb.MemoryUsage{ + Timestamp: stats.SystemNano, + WorkingSetBytes: &pb.UInt64Value{Value: stats.MemUsage}, + }, + WritableLayer: nil, + } +} + // ContainerStats returns stats of the container. If the container does not // exist, the call returns an error. func (s *Server) ContainerStats(ctx context.Context, req *pb.ContainerStatsRequest) (resp *pb.ContainerStatsResponse, err error) { @@ -16,5 +38,16 @@ func (s *Server) ContainerStats(ctx context.Context, req *pb.ContainerStatsReque recordOperation(operation, time.Now()) recordError(operation, err) }() - return nil, fmt.Errorf("not implemented") + + container := s.GetContainer(req.ContainerId) + if container == nil { + return nil, fmt.Errorf("invalid container") + } + + stats, err := s.GetContainerStats(container, &lib.ContainerStats{}) + if err != nil { + return nil, err + } + + return &pb.ContainerStatsResponse{Stats: buildContainerStats(stats, container)}, nil } diff --git a/server/container_stats_list.go b/server/container_stats_list.go index 2c564714..5749d2f6 100644 --- a/server/container_stats_list.go +++ b/server/container_stats_list.go @@ -1,9 +1,10 @@ package server import ( - "fmt" "time" + "github.com/kubernetes-incubator/cri-o/lib" + "github.com/sirupsen/logrus" "golang.org/x/net/context" pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" ) @@ -15,5 +16,37 @@ func (s *Server) ListContainerStats(ctx context.Context, req *pb.ListContainerSt recordOperation(operation, time.Now()) recordError(operation, err) }() - return nil, fmt.Errorf("not implemented") + + ctrList, err := s.ContainerServer.ListContainers() + if err != nil { + return nil, err + } + filter := req.GetFilter() + if filter != nil { + cFilter := &pb.ContainerFilter{ + Id: req.Filter.Id, + PodSandboxId: req.Filter.PodSandboxId, + LabelSelector: req.Filter.LabelSelector, + } + ctrList, err = s.filterContainerList(cFilter) + if err != nil { + return nil, err + } + } + + var allStats []*pb.ContainerStats + + for _, container := range ctrList { + stats, err := s.GetContainerStats(container, &lib.ContainerStats{}) + if err != nil { + logrus.Warn("unable to get stats for container %s", container.ID()) + continue + } + response := buildContainerStats(stats, container) + allStats = append(allStats, response) + } + + return &pb.ListContainerStatsResponse{ + Stats: allStats, + }, nil } From 50c94a9335dccf0803faed47ce9325ac1fb869f1 Mon Sep 17 00:00:00 2001 From: Yann Ramin Date: Wed, 31 Jan 2018 10:28:01 -0800 Subject: [PATCH 2/4] Specifying a filter with no filtering expressions is now idempotent Signed-off-by: Yann Ramin --- server/container_list.go | 6 ++++-- server/container_stats_list.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/container_list.go b/server/container_list.go index 42ddb4e2..85681489 100644 --- a/server/container_list.go +++ b/server/container_list.go @@ -30,7 +30,9 @@ func filterContainer(c *pb.Container, filter *pb.ContainerFilter) bool { // filterContainerList applies a protobuf-defined filter to retrieve only intended containers. Not matching // the filter is not considered an error but will return an empty response. -func (s *Server) filterContainerList(filter *pb.ContainerFilter) (ctrList []*oci.Container, err error) { +func (s *Server) filterContainerList(filter *pb.ContainerFilter, origCtrList []*oci.Container) (ctrList []*oci.Container, err error) { + ctrList = origCtrList + // Filter using container id and pod id first. if filter.Id != "" { id, err := s.CtrIDIndex().Get(filter.Id) @@ -83,7 +85,7 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque } if filter != nil { - ctrList, err = s.filterContainerList(filter) + ctrList, err = s.filterContainerList(filter, ctrList) if err != nil { return nil, err } diff --git a/server/container_stats_list.go b/server/container_stats_list.go index 5749d2f6..b6a81f6b 100644 --- a/server/container_stats_list.go +++ b/server/container_stats_list.go @@ -28,7 +28,7 @@ func (s *Server) ListContainerStats(ctx context.Context, req *pb.ListContainerSt PodSandboxId: req.Filter.PodSandboxId, LabelSelector: req.Filter.LabelSelector, } - ctrList, err = s.filterContainerList(cFilter) + ctrList, err = s.filterContainerList(cFilter, ctrList) if err != nil { return nil, err } From a2fc41358a0f852b399cd394b0448ff9845425a9 Mon Sep 17 00:00:00 2001 From: Yann Ramin Date: Fri, 2 Feb 2018 15:38:45 -0800 Subject: [PATCH 3/4] Simplify filter block Signed-off-by: Yann Ramin --- server/container_list.go | 29 +++++++++-------------------- server/container_stats_list.go | 5 +---- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/server/container_list.go b/server/container_list.go index 85681489..10a451f5 100644 --- a/server/container_list.go +++ b/server/container_list.go @@ -30,9 +30,7 @@ func filterContainer(c *pb.Container, filter *pb.ContainerFilter) bool { // filterContainerList applies a protobuf-defined filter to retrieve only intended containers. Not matching // the filter is not considered an error but will return an empty response. -func (s *Server) filterContainerList(filter *pb.ContainerFilter, origCtrList []*oci.Container) (ctrList []*oci.Container, err error) { - ctrList = origCtrList - +func (s *Server) filterContainerList(filter *pb.ContainerFilter, origCtrList []*oci.Container) []*oci.Container { // Filter using container id and pod id first. if filter.Id != "" { id, err := s.CtrIDIndex().Get(filter.Id) @@ -40,32 +38,26 @@ func (s *Server) filterContainerList(filter *pb.ContainerFilter, origCtrList []* // If we don't find a container ID with a filter, it should not // be considered an error. Log a warning and return an empty struct logrus.Warn("unable to find container ID %s", filter.Id) - return []*oci.Container{}, nil + return []*oci.Container{} } c := s.ContainerServer.GetContainer(id) if c != nil { - if filter.PodSandboxId != "" { - if c.Sandbox() == filter.PodSandboxId { - ctrList = []*oci.Container{c} - } else { - ctrList = []*oci.Container{} - } - - } else { - ctrList = []*oci.Container{c} + if filter.PodSandboxId == "" || c.Sandbox() == filter.PodSandboxId { + return []*oci.Container{} } + return []*oci.Container{c} } } else { if filter.PodSandboxId != "" { pod := s.ContainerServer.GetSandbox(filter.PodSandboxId) if pod == nil { - ctrList = []*oci.Container{} + return []*oci.Container{} } else { - ctrList = pod.Containers().List() + return pod.Containers().List() } } } - return + return origCtrList } // ListContainers lists all containers by filters. @@ -85,10 +77,7 @@ func (s *Server) ListContainers(ctx context.Context, req *pb.ListContainersReque } if filter != nil { - ctrList, err = s.filterContainerList(filter, ctrList) - if err != nil { - return nil, err - } + ctrList = s.filterContainerList(filter, ctrList) } for _, ctr := range ctrList { diff --git a/server/container_stats_list.go b/server/container_stats_list.go index b6a81f6b..558b1e82 100644 --- a/server/container_stats_list.go +++ b/server/container_stats_list.go @@ -28,10 +28,7 @@ func (s *Server) ListContainerStats(ctx context.Context, req *pb.ListContainerSt PodSandboxId: req.Filter.PodSandboxId, LabelSelector: req.Filter.LabelSelector, } - ctrList, err = s.filterContainerList(cFilter, ctrList) - if err != nil { - return nil, err - } + ctrList = s.filterContainerList(cFilter, ctrList) } var allStats []*pb.ContainerStats From 9a86dbabc2547dbc7c8938d678a5e0f293008b01 Mon Sep 17 00:00:00 2001 From: Yann Ramin Date: Tue, 6 Feb 2018 01:32:23 -0600 Subject: [PATCH 4/4] Add logging support for base condition in debug Signed-off-by: Yann Ramin --- server/container_list.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/container_list.go b/server/container_list.go index 10a451f5..63d9a825 100644 --- a/server/container_list.go +++ b/server/container_list.go @@ -42,21 +42,25 @@ func (s *Server) filterContainerList(filter *pb.ContainerFilter, origCtrList []* } c := s.ContainerServer.GetContainer(id) if c != nil { - if filter.PodSandboxId == "" || c.Sandbox() == filter.PodSandboxId { + switch { + case filter.PodSandboxId == "": + return []*oci.Container{c} + case c.Sandbox() == filter.PodSandboxId: + return []*oci.Container{c} + default: return []*oci.Container{} } - return []*oci.Container{c} } } else { if filter.PodSandboxId != "" { pod := s.ContainerServer.GetSandbox(filter.PodSandboxId) if pod == nil { return []*oci.Container{} - } else { - return pod.Containers().List() } + return pod.Containers().List() } } + logrus.Debug("no filters were applied, returning full container list") return origCtrList }