From b4a469edfe5033173e46d592d1eef37c9a8ca9b1 Mon Sep 17 00:00:00 2001 From: Yann Ramin Date: Sun, 7 Jan 2018 14:16:46 -0800 Subject: [PATCH 1/2] Add methods for listing and fetching container stats This uses the previously unusued lib/stats.go code to return data about container stats to the CRI API. As a shortcut for intial testing, I've made ListContainerStats an API composition of the ContainerStats and ListContainers APIs directly (calls the same server methods with the same context), but this should be refactored into an internal API. No data on filesystem layer usage is returned at this time. Fixes one-half of #1248 Signed-off-by: Yann Ramin --- lib/stats.go | 8 ++++---- server/container_stats.go | 33 ++++++++++++++++++++++++++++++++- server/container_stats_list.go | 31 +++++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/lib/stats.go b/lib/stats.go index 229d8409..591156e7 100644 --- a/lib/stats.go +++ b/lib/stats.go @@ -15,8 +15,8 @@ import ( type ContainerStats struct { Container string CPU float64 - cpuNano uint64 - systemNano uint64 + CPUNano uint64 + SystemNano uint64 MemUsage uint64 MemLimit uint64 MemPerc float64 @@ -29,8 +29,8 @@ type ContainerStats struct { // GetContainerStats gets the running stats for a given container func (c *ContainerServer) GetContainerStats(ctr *oci.Container, previousStats *ContainerStats) (*ContainerStats, error) { - previousCPU := previousStats.cpuNano - previousSystem := previousStats.systemNano + previousCPU := previousStats.CPUNano + previousSystem := previousStats.SystemNano libcontainerStats, err := c.LibcontainerStats(ctr) if err != nil { return nil, err diff --git a/server/container_stats.go b/server/container_stats.go index 17df31ad..8e6176af 100644 --- a/server/container_stats.go +++ b/server/container_stats.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/kubernetes-incubator/cri-o/lib" "golang.org/x/net/context" pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" ) @@ -16,5 +17,35 @@ func (s *Server) ContainerStats(ctx context.Context, req *pb.ContainerStatsReque recordOperation(operation, time.Now()) recordError(operation, err) }() - return nil, fmt.Errorf("not implemented") + + container := s.GetContainer(req.ContainerId) + if container == nil { + return nil, fmt.Errorf("invalid container") + } + + now := time.Now().UnixNano() + stats, err := s.GetContainerStats(container, &lib.ContainerStats{}) + if err != nil { + return nil, err + } + + return &pb.ContainerStatsResponse{ + &pb.ContainerStats{ + Attributes: &pb.ContainerAttributes{ + Id: req.ContainerId, + Metadata: container.Metadata(), + Labels: container.Labels(), + Annotations: container.Annotations(), + }, + Cpu: &pb.CpuUsage{ + Timestamp: now, + UsageCoreNanoSeconds: &pb.UInt64Value{stats.CPUNano + stats.SystemNano}, + }, + Memory: &pb.MemoryUsage{ + Timestamp: now, + WorkingSetBytes: &pb.UInt64Value{stats.MemUsage}, + }, + WritableLayer: nil, + }, + }, nil } diff --git a/server/container_stats_list.go b/server/container_stats_list.go index 2c564714..7c78879b 100644 --- a/server/container_stats_list.go +++ b/server/container_stats_list.go @@ -1,9 +1,9 @@ package server import ( - "fmt" "time" + "github.com/sirupsen/logrus" "golang.org/x/net/context" pb "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" ) @@ -15,5 +15,32 @@ func (s *Server) ListContainerStats(ctx context.Context, req *pb.ListContainerSt recordOperation(operation, time.Now()) recordError(operation, err) }() - return nil, fmt.Errorf("not implemented") + + // This is an inefficient method, since the container will be resolved twice, + // once by the container list code and once by the GetContainerStats call. + containers, err := s.ListContainers(ctx, &pb.ListContainersRequest{ + &pb.ContainerFilter{ + Id: req.Filter.Id, + PodSandboxId: req.Filter.PodSandboxId, + LabelSelector: req.Filter.LabelSelector, + }, + }) + if err != nil { + return nil, err + } + + var allStats []*pb.ContainerStats + + for _, container := range containers.Containers { + stats, err := s.ContainerStats(ctx, &pb.ContainerStatsRequest{ContainerId: container.Id}) + if err != nil { + logrus.Warn("unable to get stats for container %s", container.Id) + continue + } + allStats = append(allStats, stats.Stats) + } + + return &pb.ListContainerStatsResponse{ + Stats: allStats, + }, nil } From 10c5dbf061075b776434b6134a6657bac2ad1c58 Mon Sep 17 00:00:00 2001 From: Yann Ramin Date: Sun, 7 Jan 2018 18:19:11 -0800 Subject: [PATCH 2/2] Fix CPU accounting Signed-off-by: Yann Ramin --- lib/stats.go | 8 +++++--- server/container_stats.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/stats.go b/lib/stats.go index 591156e7..58fac59d 100644 --- a/lib/stats.go +++ b/lib/stats.go @@ -16,7 +16,7 @@ type ContainerStats struct { Container string CPU float64 CPUNano uint64 - SystemNano uint64 + SystemNano int64 MemUsage uint64 MemLimit uint64 MemPerc float64 @@ -38,6 +38,8 @@ func (c *ContainerServer) GetContainerStats(ctr *oci.Container, previousStats *C cgroupStats := libcontainerStats.CgroupStats stats := new(ContainerStats) stats.Container = ctr.ID() + stats.CPUNano = cgroupStats.CpuStats.CpuUsage.TotalUsage + stats.SystemNano = time.Now().UnixNano() stats.CPU = calculateCPUPercent(libcontainerStats, previousCPU, previousSystem) stats.MemUsage = cgroupStats.MemoryStats.Usage.Usage stats.MemLimit = getMemLimit(cgroupStats.MemoryStats.Usage.Limit) @@ -84,11 +86,11 @@ func getContainerNetIO(stats *libcontainer.Stats) (received uint64, transmitted return } -func calculateCPUPercent(stats *libcontainer.Stats, previousCPU, previousSystem uint64) float64 { +func calculateCPUPercent(stats *libcontainer.Stats, previousCPU uint64, previousSystem int64) float64 { var ( cpuPercent = 0.0 cpuDelta = float64(stats.CgroupStats.CpuStats.CpuUsage.TotalUsage - previousCPU) - systemDelta = float64(uint64(time.Now().UnixNano()) - previousSystem) + systemDelta = float64(uint64(time.Now().UnixNano()) - uint64(previousSystem)) ) if systemDelta > 0.0 && cpuDelta > 0.0 { // gets a ratio of container cpu usage total, multiplies it by the number of cores (4 cores running diff --git a/server/container_stats.go b/server/container_stats.go index 8e6176af..ffffe160 100644 --- a/server/container_stats.go +++ b/server/container_stats.go @@ -39,7 +39,7 @@ func (s *Server) ContainerStats(ctx context.Context, req *pb.ContainerStatsReque }, Cpu: &pb.CpuUsage{ Timestamp: now, - UsageCoreNanoSeconds: &pb.UInt64Value{stats.CPUNano + stats.SystemNano}, + UsageCoreNanoSeconds: &pb.UInt64Value{stats.CPUNano}, }, Memory: &pb.MemoryUsage{ Timestamp: now,