diff --git a/containerd/main.go b/containerd/main.go index 4c0e141..d09a406 100644 --- a/containerd/main.go +++ b/containerd/main.go @@ -14,6 +14,7 @@ import ( "google.golang.org/grpc" "github.com/Sirupsen/logrus" + "github.com/cloudfoundry/gosigar" "github.com/codegangsta/cli" "github.com/cyberdelia/go-metrics-graphite" "github.com/docker/containerd" @@ -150,20 +151,33 @@ func debugMetrics(interval time.Duration, graphiteAddr string) error { func processMetrics() { var ( - g = metrics.NewGauge() - fg = metrics.NewGauge() + g = metrics.NewGauge() + fg = metrics.NewGauge() + memg = metrics.NewGauge() ) metrics.DefaultRegistry.Register("goroutines", g) metrics.DefaultRegistry.Register("fds", fg) + metrics.DefaultRegistry.Register("memory-used", memg) + collect := func() { + // update number of goroutines + g.Update(int64(runtime.NumGoroutine())) + // collect the number of open fds + fds, err := util.GetOpenFds(os.Getpid()) + if err != nil { + logrus.WithField("error", err).Error("get open fd count") + } + fg.Update(int64(fds)) + // get the memory used + m := sigar.ProcMem{} + if err := m.Get(os.Getpid()); err != nil { + logrus.WithField("error", err).Error("get pid memory information") + } + memg.Update(int64(m.Size)) + } go func() { + collect() for range time.Tick(30 * time.Second) { - g.Update(int64(runtime.NumGoroutine())) - fds, err := util.GetOpenFds(os.Getpid()) - if err != nil { - logrus.WithField("error", err).Error("get open fd count") - continue - } - fg.Update(int64(fds)) + collect() } }() } diff --git a/supervisor/add_process.go b/supervisor/add_process.go index ab1fb4d..bc0efb9 100644 --- a/supervisor/add_process.go +++ b/supervisor/add_process.go @@ -1,6 +1,10 @@ package supervisor -import "github.com/Sirupsen/logrus" +import ( + "time" + + "github.com/Sirupsen/logrus" +) type AddProcessEvent struct { s *Supervisor @@ -9,6 +13,7 @@ type AddProcessEvent struct { // TODO: add this to worker for concurrent starts??? maybe not because of races where the container // could be stopped and removed... func (h *AddProcessEvent) Handle(e *Event) error { + start := time.Now() ci, ok := h.s.containers[e.ID] if !ok { return ErrContainerNotFound @@ -32,5 +37,6 @@ func (h *AddProcessEvent) Handle(e *Event) error { }).Error("log stdio") } h.s.processes[e.Pid].copier = l + ExecProcessTimer.UpdateSince(start) return nil } diff --git a/supervisor/create.go b/supervisor/create.go index b9e6572..41f4c8b 100644 --- a/supervisor/create.go +++ b/supervisor/create.go @@ -1,10 +1,13 @@ package supervisor +import "time" + type StartEvent struct { s *Supervisor } func (h *StartEvent) Handle(e *Event) error { + start := time.Now() container, io, err := h.s.runtime.Create(e.ID, e.BundlePath, e.Console) if err != nil { return err @@ -27,5 +30,6 @@ func (h *StartEvent) Handle(e *Event) error { task.Checkpoint = e.Checkpoint.Name } h.s.tasks <- task + ContainerCreateTimer.UpdateSince(start) return errDeferedResponse } diff --git a/supervisor/delete.go b/supervisor/delete.go index 94e4225..4651ef8 100644 --- a/supervisor/delete.go +++ b/supervisor/delete.go @@ -1,6 +1,8 @@ package supervisor import ( + "time" + "github.com/Sirupsen/logrus" "github.com/docker/containerd/runtime" ) @@ -11,6 +13,7 @@ type DeleteEvent struct { func (h *DeleteEvent) Handle(e *Event) error { if i, ok := h.s.containers[e.ID]; ok { + start := time.Now() if err := h.deleteContainer(i.container); err != nil { logrus.WithField("error", err).Error("containerd: deleting container") } @@ -27,6 +30,7 @@ func (h *DeleteEvent) Handle(e *Event) error { }) ContainersCounter.Dec(1) h.s.containerGroup.Done() + ContainerDeleteTimer.UpdateSince(start) } return nil } diff --git a/supervisor/exit.go b/supervisor/exit.go index 32a0fbb..93a06c1 100644 --- a/supervisor/exit.go +++ b/supervisor/exit.go @@ -1,12 +1,17 @@ package supervisor -import "github.com/Sirupsen/logrus" +import ( + "time" + + "github.com/Sirupsen/logrus" +) type ExitEvent struct { s *Supervisor } func (h *ExitEvent) Handle(e *Event) error { + start := time.Now() logrus.WithFields(logrus.Fields{"pid": e.Pid, "status": e.Status}). Debug("containerd: process exited") // is it the child process of a container @@ -36,6 +41,7 @@ func (h *ExitEvent) Handle(e *Event) error { stopCollect := NewEvent(StopStatsEventType) stopCollect.ID = container.ID() h.s.SendEvent(stopCollect) + ExitProcessTimer.UpdateSince(start) return nil } diff --git a/supervisor/metrics.go b/supervisor/metrics.go new file mode 100644 index 0000000..80ca4ba --- /dev/null +++ b/supervisor/metrics.go @@ -0,0 +1,27 @@ +package supervisor + +import "github.com/rcrowley/go-metrics" + +var ( + ContainerCreateTimer = metrics.NewTimer() + ContainerDeleteTimer = metrics.NewTimer() + ContainerStartTimer = metrics.NewTimer() + ContainersCounter = metrics.NewCounter() + EventSubscriberCounter = metrics.NewCounter() + EventsCounter = metrics.NewCounter() + ExecProcessTimer = metrics.NewTimer() + ExitProcessTimer = metrics.NewTimer() +) + +func Metrics() map[string]interface{} { + return map[string]interface{}{ + "container-create-time": ContainerCreateTimer, + "container-delete-time": ContainerDeleteTimer, + "container-start-time": ContainerStartTimer, + "containers": ContainersCounter, + "event-subscribers": EventSubscriberCounter, + "events": EventsCounter, + "exec-process-time": ExecProcessTimer, + "exit-process-time": ExitProcessTimer, + } +} diff --git a/supervisor/stats.go b/supervisor/stats.go index 379dab4..7f3a9ec 100644 --- a/supervisor/stats.go +++ b/supervisor/stats.go @@ -1,23 +1,5 @@ package supervisor -import "github.com/rcrowley/go-metrics" - -var ( - ContainerStartTimer = metrics.NewTimer() - ContainersCounter = metrics.NewCounter() - EventsCounter = metrics.NewCounter() - EventSubscriberCounter = metrics.NewCounter() -) - -func Metrics() map[string]interface{} { - return map[string]interface{}{ - "container-start-time": ContainerStartTimer, - "containers": ContainersCounter, - "events": EventsCounter, - "events-subscribers": EventSubscriberCounter, - } -} - type StatsEvent struct { s *Supervisor }