Add device path to metrics

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
This commit is contained in:
Michael Crosby 2017-03-21 16:52:16 -07:00
parent c90e0c94a5
commit d219b47f65
5 changed files with 91 additions and 20 deletions

View file

@ -1,6 +1,6 @@
github.com/crosbymichael/go-runc bd9aef7cf4402a3a8728e3ef83dcca6a5a1be899
github.com/crosbymichael/console 4bf9d88357031b516b3794a2594b6d060a29c59c
github.com/crosbymichael/cgroups 66fd96cb5fc92fdcd32b61518b2619d489784256
github.com/crosbymichael/cgroups 74ce513f69d6dbdb355668b4b4ca7ac3d0886664
github.com/docker/go-metrics 8fd5772bf1584597834c6f7961a530f06cbfbb87
github.com/prometheus/client_golang v0.8.0
github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6

View file

@ -8,6 +8,7 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
specs "github.com/opencontainers/runtime-spec/specs-go"
)
@ -104,15 +105,21 @@ func (b *blkioController) Stat(path string, stats *Stats) error {
},
)
}
devices, err := getDevices("/dev")
if err != nil {
return err
}
for _, t := range settings {
if err := b.readEntry(path, t.name, t.entry); err != nil {
if err := b.readEntry(devices, path, t.name, t.entry); err != nil {
return err
}
}
return nil
}
func (b *blkioController) readEntry(path, name string, entry *[]BlkioEntry) error {
func (b *blkioController) readEntry(devices map[deviceKey]string, path, name string, entry *[]BlkioEntry) error {
f, err := os.Open(filepath.Join(b.Path(path), fmt.Sprintf("blkio.%s", name)))
if err != nil {
return err
@ -152,6 +159,7 @@ func (b *blkioController) readEntry(path, name string, entry *[]BlkioEntry) erro
return err
}
*entry = append(*entry, BlkioEntry{
Device: devices[deviceKey{major, minor}],
Major: major,
Minor: minor,
Op: op,
@ -160,6 +168,7 @@ func (b *blkioController) readEntry(path, name string, entry *[]BlkioEntry) erro
}
return nil
}
func createBlkioSettings(blkio *specs.LinuxBlockIO) []blkioSettings {
settings := []blkioSettings{
{
@ -251,3 +260,64 @@ func throttleddev(v interface{}) []byte {
func splitBlkioStatLine(r rune) bool {
return r == ' ' || r == ':'
}
type deviceKey struct {
major, minor uint64
}
// getDevices makes a best effort attempt to read all the devices into a map
// keyed by major and minor number. Since devices may be mapped multiple times,
// we err on taking the first occurrence.
func getDevices(path string) (map[deviceKey]string, error) {
// TODO(stevvooe): We are ignoring lots of errors. It might be kind of
// challenging to debug this if we aren't mapping devices correctly.
// Consider logging these errors.
devices := map[deviceKey]string{}
if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
switch {
case fi.IsDir():
switch fi.Name() {
case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts":
return filepath.SkipDir
default:
return nil
}
case fi.Name() == "console":
return nil
default:
if fi.Mode()&os.ModeDevice == 0 {
// skip non-devices
return nil
}
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return fmt.Errorf("%s: unable to convert to system stat", p)
}
key := deviceKey{major(st.Rdev), minor(st.Rdev)}
if _, ok := devices[key]; ok {
return nil // skip it if we have already populated the path.
}
devices[key] = p
}
return nil
}); err != nil {
return nil, err
}
return devices, nil
}
func major(devNumber uint64) uint64 {
return (devNumber >> 8) & 0xfff
}
func minor(devNumber uint64) uint64 {
return (devNumber & 0xff) | ((devNumber >> 12) & 0xfff00)
}

View file

@ -12,7 +12,7 @@ var blkioMetrics = []*metric{
help: "The blkio io merged recursive",
unit: metrics.Total,
vt: prometheus.GaugeValue,
labels: []string{"op", "major", "minor"},
labels: []string{"op", "device", "major", "minor"},
getValues: func(stats *cgroups.Stats) []value {
if stats.Blkio == nil {
return nil
@ -25,7 +25,7 @@ var blkioMetrics = []*metric{
help: "The blkio io queued recursive",
unit: metrics.Total,
vt: prometheus.GaugeValue,
labels: []string{"op", "major", "minor"},
labels: []string{"op", "device", "major", "minor"},
getValues: func(stats *cgroups.Stats) []value {
if stats.Blkio == nil {
return nil
@ -38,7 +38,7 @@ var blkioMetrics = []*metric{
help: "The blkio io service bytes recursive",
unit: metrics.Bytes,
vt: prometheus.GaugeValue,
labels: []string{"op", "major", "minor"},
labels: []string{"op", "device", "major", "minor"},
getValues: func(stats *cgroups.Stats) []value {
if stats.Blkio == nil {
return nil
@ -51,7 +51,7 @@ var blkioMetrics = []*metric{
help: "The blkio io servie time recursive",
unit: metrics.Total,
vt: prometheus.GaugeValue,
labels: []string{"op", "major", "minor"},
labels: []string{"op", "device", "major", "minor"},
getValues: func(stats *cgroups.Stats) []value {
if stats.Blkio == nil {
return nil
@ -64,7 +64,7 @@ var blkioMetrics = []*metric{
help: "The blkio io servied recursive",
unit: metrics.Total,
vt: prometheus.GaugeValue,
labels: []string{"op", "major", "minor"},
labels: []string{"op", "device", "major", "minor"},
getValues: func(stats *cgroups.Stats) []value {
if stats.Blkio == nil {
return nil
@ -77,7 +77,7 @@ var blkioMetrics = []*metric{
help: "The blkio io time recursive",
unit: metrics.Total,
vt: prometheus.GaugeValue,
labels: []string{"op", "major", "minor"},
labels: []string{"op", "device", "major", "minor"},
getValues: func(stats *cgroups.Stats) []value {
if stats.Blkio == nil {
return nil
@ -90,7 +90,7 @@ var blkioMetrics = []*metric{
help: "The blkio sectors recursive",
unit: metrics.Total,
vt: prometheus.GaugeValue,
labels: []string{"op", "major", "minor"},
labels: []string{"op", "device", "major", "minor"},
getValues: func(stats *cgroups.Stats) []value {
if stats.Blkio == nil {
return nil

View file

@ -59,7 +59,7 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
func (c *Collector) collect(id string, cg cgroups.Cgroup, ch chan<- prometheus.Metric, wg *sync.WaitGroup) {
defer wg.Done()
stats, err := cg.Stat()
stats, err := cg.Stat(cgroups.IgnoreNotExist)
if err != nil {
logrus.WithError(err).Errorf("stat cgroup %s", id)
return
@ -92,7 +92,7 @@ func blkioValues(l []cgroups.BlkioEntry) []value {
for _, e := range l {
out = append(out, value{
v: float64(e.Value),
l: []string{e.Op, strconv.FormatUint(e.Major, 10), strconv.FormatUint(e.Minor, 10)},
l: []string{e.Op, e.Device, strconv.FormatUint(e.Major, 10), strconv.FormatUint(e.Minor, 10)},
})
}
return out

View file

@ -102,6 +102,7 @@ type BlkioStat struct {
type BlkioEntry struct {
Op string
Device string
Major uint64
Minor uint64
Value uint64