Add fd output to debug metrics

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

containerd/main.go

@@ -12,23 +12,20 @@ import (
 	"github.com/codegangsta/cli"
 	"github.com/docker/containerd"
 	"github.com/docker/containerd/api/v1"
+	"github.com/docker/containerd/util"
 	"github.com/rcrowley/go-metrics"
 )
 
 const Usage = `High performance conatiner daemon`
 
-func main() {
-	app := cli.NewApp()
-	app.Name = "containerd"
-	app.Version = containerd.Version
-	app.Usage = Usage
-	app.Authors = []cli.Author{
+var authors = []cli.Author{
 	{
 		Name:  "@crosbymichael",
 		Email: "crosbymichael@gmail.com",
 	},
 }
-	app.Flags = []cli.Flag{
+
+var daemonFlags = []cli.Flag{
 	cli.StringFlag{
 		Name:  "id",
 		Value: getDefaultID(),
@@ -53,24 +50,24 @@ func main() {
 		Value: 10,
 		Usage: "set the concurrency level for tasks",
 	},
+	cli.DurationFlag{
+		Name:  "metrics-interval",
+		Value: 60 * time.Second,
+		Usage: "interval for flushing metrics to the store",
+	},
 }
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "containerd"
+	app.Version = containerd.Version
+	app.Usage = Usage
+	app.Authors = authors
+	app.Flags = daemonFlags
 	app.Before = func(context *cli.Context) error {
 		if context.GlobalBool("debug") {
 			logrus.SetLevel(logrus.DebugLevel)
-			l := log.New(os.Stdout, "[containerd] ", log.LstdFlags)
-			goRoutineCounter := metrics.NewGauge()
-			metrics.DefaultRegistry.Register("goroutines", goRoutineCounter)
-			for name, m := range containerd.Metrics() {
-				if err := metrics.DefaultRegistry.Register(name, m); err != nil {
-					return err
-				}
-			}
-			go func() {
-				for range time.Tick(30 * time.Second) {
-					goRoutineCounter.Update(int64(runtime.NumGoroutine()))
-				}
-			}()
-			go metrics.Log(metrics.DefaultRegistry, 60*time.Second, l)
+			return debugMetrics(context.GlobalDuration("metrics-interval"))
 		}
 		return nil
 	}
@@ -89,6 +86,38 @@ func main() {
 	}
 }
 
+func debugMetrics(interval time.Duration) error {
+	for name, m := range containerd.Metrics() {
+		if err := metrics.DefaultRegistry.Register(name, m); err != nil {
+			return err
+		}
+	}
+	processMetrics()
+	l := log.New(os.Stdout, "[containerd] ", log.LstdFlags)
+	go metrics.Log(metrics.DefaultRegistry, interval, l)
+	return nil
+}
+
+func processMetrics() {
+	var (
+		g  = metrics.NewGauge()
+		fg = metrics.NewGauge()
+	)
+	metrics.DefaultRegistry.Register("goroutines", g)
+	metrics.DefaultRegistry.Register("fds", fg)
+	go func() {
+		for range time.Tick(30 * time.Second) {
+			g.Update(int64(runtime.NumGoroutine()))
+			fds, err := util.GetOpenFds(os.Getpid())
+			if err != nil {
+				logrus.WithField("error", err).Error("get open fd count")
+				continue
+			}
+			fg.Update(int64(fds))
+		}
+	}()
+}
+
 func daemon(id, stateDir string, concurrency, bufferSize int) error {
 	tasks := make(chan *containerd.StartTask, concurrency*100)
 	supervisor, err := containerd.NewSupervisor(id, stateDir, tasks)

util/fds.go

@@ -0,0 +1,16 @@
+package util
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+)
+
+// GetOpenFds returns the number of open fds for the process provided by pid
+func GetOpenFds(pid int) (int, error) {
+	dirs, err := ioutil.ReadDir(filepath.Join("/proc", strconv.Itoa(pid), "fd"))
+	if err != nil {
+		return -1, err
+	}
+	return len(dirs), nil
+}