Add more metrics collection

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

containerd/main.go

@@ -14,6 +14,7 @@ import (
 	"google.golang.org/grpc"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/cloudfoundry/gosigar"
 	"github.com/codegangsta/cli"
 	"github.com/cyberdelia/go-metrics-graphite"
 	"github.com/docker/containerd"
@@ -150,20 +151,33 @@ func debugMetrics(interval time.Duration, graphiteAddr string) error {
 
 func processMetrics() {
 	var (
-		g  = metrics.NewGauge()
-		fg = metrics.NewGauge()
+		g    = metrics.NewGauge()
+		fg   = metrics.NewGauge()
+		memg = metrics.NewGauge()
 	)
 	metrics.DefaultRegistry.Register("goroutines", g)
 	metrics.DefaultRegistry.Register("fds", fg)
+	metrics.DefaultRegistry.Register("memory-used", memg)
+	collect := func() {
+		// update number of goroutines
+		g.Update(int64(runtime.NumGoroutine()))
+		// collect the number of open fds
+		fds, err := util.GetOpenFds(os.Getpid())
+		if err != nil {
+			logrus.WithField("error", err).Error("get open fd count")
+		}
+		fg.Update(int64(fds))
+		// get the memory used
+		m := sigar.ProcMem{}
+		if err := m.Get(os.Getpid()); err != nil {
+			logrus.WithField("error", err).Error("get pid memory information")
+		}
+		memg.Update(int64(m.Size))
+	}
 	go func() {
+		collect()
 		for range time.Tick(30 * time.Second) {
-			g.Update(int64(runtime.NumGoroutine()))
-			fds, err := util.GetOpenFds(os.Getpid())
-			if err != nil {
-				logrus.WithField("error", err).Error("get open fd count")
-				continue
-			}
-			fg.Update(int64(fds))
+			collect()
 		}
 	}()
 }

supervisor/add_process.go

@@ -1,6 +1,10 @@
 package supervisor
 
-import "github.com/Sirupsen/logrus"
+import (
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
 
 type AddProcessEvent struct {
 	s *Supervisor
@@ -9,6 +13,7 @@ type AddProcessEvent struct {
 // TODO: add this to worker for concurrent starts???  maybe not because of races where the container
 // could be stopped and removed...
 func (h *AddProcessEvent) Handle(e *Event) error {
+	start := time.Now()
 	ci, ok := h.s.containers[e.ID]
 	if !ok {
 		return ErrContainerNotFound
@@ -32,5 +37,6 @@ func (h *AddProcessEvent) Handle(e *Event) error {
 		}).Error("log stdio")
 	}
 	h.s.processes[e.Pid].copier = l
+	ExecProcessTimer.UpdateSince(start)
 	return nil
 }

supervisor/create.go

@@ -1,10 +1,13 @@
 package supervisor
 
+import "time"
+
 type StartEvent struct {
 	s *Supervisor
 }
 
 func (h *StartEvent) Handle(e *Event) error {
+	start := time.Now()
 	container, io, err := h.s.runtime.Create(e.ID, e.BundlePath, e.Console)
 	if err != nil {
 		return err
@@ -27,5 +30,6 @@ func (h *StartEvent) Handle(e *Event) error {
 		task.Checkpoint = e.Checkpoint.Name
 	}
 	h.s.tasks <- task
+	ContainerCreateTimer.UpdateSince(start)
 	return errDeferedResponse
 }

supervisor/delete.go

@@ -1,6 +1,8 @@
 package supervisor
 
 import (
+	"time"
+
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/containerd/runtime"
 )
@@ -11,6 +13,7 @@ type DeleteEvent struct {
 
 func (h *DeleteEvent) Handle(e *Event) error {
 	if i, ok := h.s.containers[e.ID]; ok {
+		start := time.Now()
 		if err := h.deleteContainer(i.container); err != nil {
 			logrus.WithField("error", err).Error("containerd: deleting container")
 		}
@@ -27,6 +30,7 @@ func (h *DeleteEvent) Handle(e *Event) error {
 		})
 		ContainersCounter.Dec(1)
 		h.s.containerGroup.Done()
+		ContainerDeleteTimer.UpdateSince(start)
 	}
 	return nil
 }

supervisor/exit.go

@@ -1,12 +1,17 @@
 package supervisor
 
-import "github.com/Sirupsen/logrus"
+import (
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
 
 type ExitEvent struct {
 	s *Supervisor
 }
 
 func (h *ExitEvent) Handle(e *Event) error {
+	start := time.Now()
 	logrus.WithFields(logrus.Fields{"pid": e.Pid, "status": e.Status}).
 		Debug("containerd: process exited")
 	// is it the child process of a container
@@ -36,6 +41,7 @@ func (h *ExitEvent) Handle(e *Event) error {
 	stopCollect := NewEvent(StopStatsEventType)
 	stopCollect.ID = container.ID()
 	h.s.SendEvent(stopCollect)
+	ExitProcessTimer.UpdateSince(start)
 	return nil
 }
 

supervisor/metrics.go

@@ -0,0 +1,27 @@
+package supervisor
+
+import "github.com/rcrowley/go-metrics"
+
+var (
+	ContainerCreateTimer   = metrics.NewTimer()
+	ContainerDeleteTimer   = metrics.NewTimer()
+	ContainerStartTimer    = metrics.NewTimer()
+	ContainersCounter      = metrics.NewCounter()
+	EventSubscriberCounter = metrics.NewCounter()
+	EventsCounter          = metrics.NewCounter()
+	ExecProcessTimer       = metrics.NewTimer()
+	ExitProcessTimer       = metrics.NewTimer()
+)
+
+func Metrics() map[string]interface{} {
+	return map[string]interface{}{
+		"container-create-time": ContainerCreateTimer,
+		"container-delete-time": ContainerDeleteTimer,
+		"container-start-time":  ContainerStartTimer,
+		"containers":            ContainersCounter,
+		"event-subscribers":     EventSubscriberCounter,
+		"events":                EventsCounter,
+		"exec-process-time":     ExecProcessTimer,
+		"exit-process-time":     ExitProcessTimer,
+	}
+}

supervisor/stats.go

@@ -1,23 +1,5 @@
 package supervisor
 
-import "github.com/rcrowley/go-metrics"
-
-var (
-	ContainerStartTimer    = metrics.NewTimer()
-	ContainersCounter      = metrics.NewCounter()
-	EventsCounter          = metrics.NewCounter()
-	EventSubscriberCounter = metrics.NewCounter()
-)
-
-func Metrics() map[string]interface{} {
-	return map[string]interface{}{
-		"container-start-time": ContainerStartTimer,
-		"containers":           ContainersCounter,
-		"events":               EventsCounter,
-		"events-subscribers":   EventSubscriberCounter,
-	}
-}
-
 type StatsEvent struct {
 	s *Supervisor
 }