Switch to new vendor directory layout

Fixes #113

Signed-off-by: Marcos Lilljedahl <marcosnils@gmail.com>

vendor/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go

@@ -0,0 +1,64 @@
+// +build linux
+
+package cgroups
+
+import (
+	"fmt"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type Manager interface {
+	// Apply cgroup configuration to the process with the specified pid
+	Apply(pid int) error
+
+	// Returns the PIDs inside the cgroup set
+	GetPids() ([]int, error)
+
+	// Returns the PIDs inside the cgroup set & all sub-cgroups
+	GetAllPids() ([]int, error)
+
+	// Returns statistics for the cgroup set
+	GetStats() (*Stats, error)
+
+	// Toggles the freezer cgroup according with specified state
+	Freeze(state configs.FreezerState) error
+
+	// Destroys the cgroup set
+	Destroy() error
+
+	// NewCgroupManager() and LoadCgroupManager() require following attributes:
+	// 	Paths   map[string]string
+	// 	Cgroups *cgroups.Cgroup
+	// Paths maps cgroup subsystem to path at which it is mounted.
+	// Cgroups specifies specific cgroup settings for the various subsystems
+
+	// Returns cgroup paths to save in a state file and to be able to
+	// restore the object later.
+	GetPaths() map[string]string
+
+	// Set the cgroup as configured.
+	Set(container *configs.Config) error
+}
+
+type NotFoundError struct {
+	Subsystem string
+}
+
+func (e *NotFoundError) Error() string {
+	return fmt.Sprintf("mountpoint for %s not found", e.Subsystem)
+}
+
+func NewNotFoundError(sub string) error {
+	return &NotFoundError{
+		Subsystem: sub,
+	}
+}
+
+func IsNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, ok := err.(*NotFoundError)
+	return ok
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/cgroups_test.go

@@ -0,0 +1,18 @@
+// +build linux
+
+package cgroups
+
+import (
+	"testing"
+)
+
+func TestParseCgroups(t *testing.T) {
+	cgroups, err := ParseCgroupFile("/proc/self/cgroup")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := cgroups["cpu"]; !ok {
+		t.Fail()
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/cgroups_unsupported.go

@@ -0,0 +1,3 @@
+// +build !linux
+
+package cgroups

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go

@@ -0,0 +1,397 @@
+// +build linux
+
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
+)
+
+var (
+	subsystems = subsystemSet{
+		&CpusetGroup{},
+		&DevicesGroup{},
+		&MemoryGroup{},
+		&CpuGroup{},
+		&CpuacctGroup{},
+		&PidsGroup{},
+		&BlkioGroup{},
+		&HugetlbGroup{},
+		&NetClsGroup{},
+		&NetPrioGroup{},
+		&PerfEventGroup{},
+		&FreezerGroup{},
+		&NameGroup{GroupName: "name=systemd", Join: true},
+	}
+	CgroupProcesses  = "cgroup.procs"
+	HugePageSizes, _ = cgroups.GetHugePageSize()
+)
+
+var errSubsystemDoesNotExist = errors.New("cgroup: subsystem does not exist")
+
+type subsystemSet []subsystem
+
+func (s subsystemSet) Get(name string) (subsystem, error) {
+	for _, ss := range s {
+		if ss.Name() == name {
+			return ss, nil
+		}
+	}
+	return nil, errSubsystemDoesNotExist
+}
+
+type subsystem interface {
+	// Name returns the name of the subsystem.
+	Name() string
+	// Returns the stats, as 'stats', corresponding to the cgroup under 'path'.
+	GetStats(path string, stats *cgroups.Stats) error
+	// Removes the cgroup represented by 'cgroupData'.
+	Remove(*cgroupData) error
+	// Creates and joins the cgroup represented by 'cgroupData'.
+	Apply(*cgroupData) error
+	// Set the cgroup represented by cgroup.
+	Set(path string, cgroup *configs.Cgroup) error
+}
+
+type Manager struct {
+	mu      sync.Mutex
+	Cgroups *configs.Cgroup
+	Paths   map[string]string
+}
+
+// The absolute path to the root of the cgroup hierarchies.
+var cgroupRootLock sync.Mutex
+var cgroupRoot string
+
+// Gets the cgroupRoot.
+func getCgroupRoot() (string, error) {
+	cgroupRootLock.Lock()
+	defer cgroupRootLock.Unlock()
+
+	if cgroupRoot != "" {
+		return cgroupRoot, nil
+	}
+
+	root, err := cgroups.FindCgroupMountpointDir()
+	if err != nil {
+		return "", err
+	}
+
+	if _, err := os.Stat(root); err != nil {
+		return "", err
+	}
+
+	cgroupRoot = root
+	return cgroupRoot, nil
+}
+
+type cgroupData struct {
+	root      string
+	innerPath string
+	config    *configs.Cgroup
+	pid       int
+}
+
+func (m *Manager) Apply(pid int) (err error) {
+	if m.Cgroups == nil {
+		return nil
+	}
+
+	var c = m.Cgroups
+
+	d, err := getCgroupData(m.Cgroups, pid)
+	if err != nil {
+		return err
+	}
+
+	if c.Paths != nil {
+		paths := make(map[string]string)
+		for name, path := range c.Paths {
+			_, err := d.path(name)
+			if err != nil {
+				if cgroups.IsNotFound(err) {
+					continue
+				}
+				return err
+			}
+			paths[name] = path
+		}
+		m.Paths = paths
+		return cgroups.EnterPid(m.Paths, pid)
+	}
+
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	paths := make(map[string]string)
+	for _, sys := range subsystems {
+		if err := sys.Apply(d); err != nil {
+			return err
+		}
+		// TODO: Apply should, ideally, be reentrant or be broken up into a separate
+		// create and join phase so that the cgroup hierarchy for a container can be
+		// created then join consists of writing the process pids to cgroup.procs
+		p, err := d.path(sys.Name())
+		if err != nil {
+			if cgroups.IsNotFound(err) {
+				continue
+			}
+			return err
+		}
+		paths[sys.Name()] = p
+	}
+	m.Paths = paths
+	return nil
+}
+
+func (m *Manager) Destroy() error {
+	if m.Cgroups.Paths != nil {
+		return nil
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if err := cgroups.RemovePaths(m.Paths); err != nil {
+		return err
+	}
+	m.Paths = make(map[string]string)
+	return nil
+}
+
+func (m *Manager) GetPaths() map[string]string {
+	m.mu.Lock()
+	paths := m.Paths
+	m.mu.Unlock()
+	return paths
+}
+
+func (m *Manager) GetStats() (*cgroups.Stats, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	stats := cgroups.NewStats()
+	for name, path := range m.Paths {
+		sys, err := subsystems.Get(name)
+		if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) {
+			continue
+		}
+		if err := sys.GetStats(path, stats); err != nil {
+			return nil, err
+		}
+	}
+	return stats, nil
+}
+
+func (m *Manager) Set(container *configs.Config) error {
+	for _, sys := range subsystems {
+		// Generate fake cgroup data.
+		d, err := getCgroupData(container.Cgroups, -1)
+		if err != nil {
+			return err
+		}
+		// Get the path, but don't error out if the cgroup wasn't found.
+		path, err := d.path(sys.Name())
+		if err != nil && !cgroups.IsNotFound(err) {
+			return err
+		}
+
+		if err := sys.Set(path, container.Cgroups); err != nil {
+			return err
+		}
+	}
+
+	if m.Paths["cpu"] != "" {
+		if err := CheckCpushares(m.Paths["cpu"], container.Cgroups.Resources.CpuShares); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Freeze toggles the container's freezer cgroup depending on the state
+// provided
+func (m *Manager) Freeze(state configs.FreezerState) error {
+	d, err := getCgroupData(m.Cgroups, 0)
+	if err != nil {
+		return err
+	}
+	dir, err := d.path("freezer")
+	if err != nil {
+		return err
+	}
+	prevState := m.Cgroups.Resources.Freezer
+	m.Cgroups.Resources.Freezer = state
+	freezer, err := subsystems.Get("freezer")
+	if err != nil {
+		return err
+	}
+	err = freezer.Set(dir, m.Cgroups)
+	if err != nil {
+		m.Cgroups.Resources.Freezer = prevState
+		return err
+	}
+	return nil
+}
+
+func (m *Manager) GetPids() ([]int, error) {
+	dir, err := getCgroupPath(m.Cgroups)
+	if err != nil {
+		return nil, err
+	}
+	return cgroups.GetPids(dir)
+}
+
+func (m *Manager) GetAllPids() ([]int, error) {
+	dir, err := getCgroupPath(m.Cgroups)
+	if err != nil {
+		return nil, err
+	}
+	return cgroups.GetAllPids(dir)
+}
+
+func getCgroupPath(c *configs.Cgroup) (string, error) {
+	d, err := getCgroupData(c, 0)
+	if err != nil {
+		return "", err
+	}
+
+	return d.path("devices")
+}
+
+func getCgroupData(c *configs.Cgroup, pid int) (*cgroupData, error) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		return nil, err
+	}
+
+	if (c.Name != "" || c.Parent != "") && c.Path != "" {
+		return nil, fmt.Errorf("cgroup: either Path or Name and Parent should be used")
+	}
+
+	// XXX: Do not remove this code. Path safety is important! -- cyphar
+	cgPath := libcontainerUtils.CleanPath(c.Path)
+	cgParent := libcontainerUtils.CleanPath(c.Parent)
+	cgName := libcontainerUtils.CleanPath(c.Name)
+
+	innerPath := cgPath
+	if innerPath == "" {
+		innerPath = filepath.Join(cgParent, cgName)
+	}
+
+	return &cgroupData{
+		root:      root,
+		innerPath: innerPath,
+		config:    c,
+		pid:       pid,
+	}, nil
+}
+
+func (raw *cgroupData) parentPath(subsystem, mountpoint, root string) (string, error) {
+	// Use GetThisCgroupDir instead of GetInitCgroupDir, because the creating
+	// process could in container and shared pid namespace with host, and
+	// /proc/1/cgroup could point to whole other world of cgroups.
+	initPath, err := cgroups.GetThisCgroupDir(subsystem)
+	if err != nil {
+		return "", err
+	}
+	// This is needed for nested containers, because in /proc/self/cgroup we
+	// see pathes from host, which don't exist in container.
+	relDir, err := filepath.Rel(root, initPath)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(mountpoint, relDir), nil
+}
+
+func (raw *cgroupData) path(subsystem string) (string, error) {
+	mnt, root, err := cgroups.FindCgroupMountpointAndRoot(subsystem)
+	// If we didn't mount the subsystem, there is no point we make the path.
+	if err != nil {
+		return "", err
+	}
+
+	// If the cgroup name/path is absolute do not look relative to the cgroup of the init process.
+	if filepath.IsAbs(raw.innerPath) {
+		// Sometimes subsystems can be mounted togethger as 'cpu,cpuacct'.
+		return filepath.Join(raw.root, filepath.Base(mnt), raw.innerPath), nil
+	}
+
+	parentPath, err := raw.parentPath(subsystem, mnt, root)
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(parentPath, raw.innerPath), nil
+}
+
+func (raw *cgroupData) join(subsystem string) (string, error) {
+	path, err := raw.path(subsystem)
+	if err != nil {
+		return "", err
+	}
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return "", err
+	}
+	if err := writeFile(path, CgroupProcesses, strconv.Itoa(raw.pid)); err != nil {
+		return "", err
+	}
+	return path, nil
+}
+
+func writeFile(dir, file, data string) error {
+	// Normally dir should not be empty, one case is that cgroup subsystem
+	// is not mounted, we will get empty dir, and we want it fail here.
+	if dir == "" {
+		return fmt.Errorf("no such directory for %s.", file)
+	}
+	return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700)
+}
+
+func readFile(dir, file string) (string, error) {
+	data, err := ioutil.ReadFile(filepath.Join(dir, file))
+	return string(data), err
+}
+
+func removePath(p string, err error) error {
+	if err != nil {
+		return err
+	}
+	if p != "" {
+		return os.RemoveAll(p)
+	}
+	return nil
+}
+
+func CheckCpushares(path string, c int64) error {
+	var cpuShares int64
+
+	if c == 0 {
+		return nil
+	}
+
+	fd, err := os.Open(filepath.Join(path, "cpu.shares"))
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	_, err = fmt.Fscanf(fd, "%d", &cpuShares)
+	if err != nil && err != io.EOF {
+		return err
+	}
+
+	if c > cpuShares {
+		return fmt.Errorf("The maximum allowed cpu-shares is %d", cpuShares)
+	} else if c < cpuShares {
+		return fmt.Errorf("The minimum allowed cpu-shares is %d", cpuShares)
+	}
+
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw_test.go

@@ -0,0 +1,272 @@
+// +build linux
+
+package fs
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func TestInvalidCgroupPath(t *testing.T) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		t.Errorf("couldn't get cgroup root: %v", err)
+	}
+
+	config := &configs.Cgroup{
+		Path: "../../../../../../../../../../some/path",
+	}
+
+	data, err := getCgroupData(config, 0)
+	if err != nil {
+		t.Errorf("couldn't get cgroup data: %v", err)
+	}
+
+	// Make sure the final innerPath doesn't go outside the cgroup mountpoint.
+	if strings.HasPrefix(data.innerPath, "..") {
+		t.Errorf("SECURITY: cgroup innerPath is outside cgroup mountpoint!")
+	}
+
+	// Double-check, using an actual cgroup.
+	deviceRoot := filepath.Join(root, "devices")
+	devicePath, err := data.path("devices")
+	if err != nil {
+		t.Errorf("couldn't get cgroup path: %v", err)
+	}
+	if !strings.HasPrefix(devicePath, deviceRoot) {
+		t.Errorf("SECURITY: cgroup path() is outside cgroup mountpoint!")
+	}
+}
+
+func TestInvalidAbsoluteCgroupPath(t *testing.T) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		t.Errorf("couldn't get cgroup root: %v", err)
+	}
+
+	config := &configs.Cgroup{
+		Path: "/../../../../../../../../../../some/path",
+	}
+
+	data, err := getCgroupData(config, 0)
+	if err != nil {
+		t.Errorf("couldn't get cgroup data: %v", err)
+	}
+
+	// Make sure the final innerPath doesn't go outside the cgroup mountpoint.
+	if strings.HasPrefix(data.innerPath, "..") {
+		t.Errorf("SECURITY: cgroup innerPath is outside cgroup mountpoint!")
+	}
+
+	// Double-check, using an actual cgroup.
+	deviceRoot := filepath.Join(root, "devices")
+	devicePath, err := data.path("devices")
+	if err != nil {
+		t.Errorf("couldn't get cgroup path: %v", err)
+	}
+	if !strings.HasPrefix(devicePath, deviceRoot) {
+		t.Errorf("SECURITY: cgroup path() is outside cgroup mountpoint!")
+	}
+}
+
+// XXX: Remove me after we get rid of configs.Cgroup.Name and configs.Cgroup.Parent.
+func TestInvalidCgroupParent(t *testing.T) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		t.Errorf("couldn't get cgroup root: %v", err)
+	}
+
+	config := &configs.Cgroup{
+		Parent: "../../../../../../../../../../some/path",
+		Name:   "name",
+	}
+
+	data, err := getCgroupData(config, 0)
+	if err != nil {
+		t.Errorf("couldn't get cgroup data: %v", err)
+	}
+
+	// Make sure the final innerPath doesn't go outside the cgroup mountpoint.
+	if strings.HasPrefix(data.innerPath, "..") {
+		t.Errorf("SECURITY: cgroup innerPath is outside cgroup mountpoint!")
+	}
+
+	// Double-check, using an actual cgroup.
+	deviceRoot := filepath.Join(root, "devices")
+	devicePath, err := data.path("devices")
+	if err != nil {
+		t.Errorf("couldn't get cgroup path: %v", err)
+	}
+	if !strings.HasPrefix(devicePath, deviceRoot) {
+		t.Errorf("SECURITY: cgroup path() is outside cgroup mountpoint!")
+	}
+}
+
+// XXX: Remove me after we get rid of configs.Cgroup.Name and configs.Cgroup.Parent.
+func TestInvalidAbsoluteCgroupParent(t *testing.T) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		t.Errorf("couldn't get cgroup root: %v", err)
+	}
+
+	config := &configs.Cgroup{
+		Parent: "/../../../../../../../../../../some/path",
+		Name:   "name",
+	}
+
+	data, err := getCgroupData(config, 0)
+	if err != nil {
+		t.Errorf("couldn't get cgroup data: %v", err)
+	}
+
+	// Make sure the final innerPath doesn't go outside the cgroup mountpoint.
+	if strings.HasPrefix(data.innerPath, "..") {
+		t.Errorf("SECURITY: cgroup innerPath is outside cgroup mountpoint!")
+	}
+
+	// Double-check, using an actual cgroup.
+	deviceRoot := filepath.Join(root, "devices")
+	devicePath, err := data.path("devices")
+	if err != nil {
+		t.Errorf("couldn't get cgroup path: %v", err)
+	}
+	if !strings.HasPrefix(devicePath, deviceRoot) {
+		t.Errorf("SECURITY: cgroup path() is outside cgroup mountpoint!")
+	}
+}
+
+// XXX: Remove me after we get rid of configs.Cgroup.Name and configs.Cgroup.Parent.
+func TestInvalidCgroupName(t *testing.T) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		t.Errorf("couldn't get cgroup root: %v", err)
+	}
+
+	config := &configs.Cgroup{
+		Parent: "parent",
+		Name:   "../../../../../../../../../../some/path",
+	}
+
+	data, err := getCgroupData(config, 0)
+	if err != nil {
+		t.Errorf("couldn't get cgroup data: %v", err)
+	}
+
+	// Make sure the final innerPath doesn't go outside the cgroup mountpoint.
+	if strings.HasPrefix(data.innerPath, "..") {
+		t.Errorf("SECURITY: cgroup innerPath is outside cgroup mountpoint!")
+	}
+
+	// Double-check, using an actual cgroup.
+	deviceRoot := filepath.Join(root, "devices")
+	devicePath, err := data.path("devices")
+	if err != nil {
+		t.Errorf("couldn't get cgroup path: %v", err)
+	}
+	if !strings.HasPrefix(devicePath, deviceRoot) {
+		t.Errorf("SECURITY: cgroup path() is outside cgroup mountpoint!")
+	}
+
+}
+
+// XXX: Remove me after we get rid of configs.Cgroup.Name and configs.Cgroup.Parent.
+func TestInvalidAbsoluteCgroupName(t *testing.T) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		t.Errorf("couldn't get cgroup root: %v", err)
+	}
+
+	config := &configs.Cgroup{
+		Parent: "parent",
+		Name:   "/../../../../../../../../../../some/path",
+	}
+
+	data, err := getCgroupData(config, 0)
+	if err != nil {
+		t.Errorf("couldn't get cgroup data: %v", err)
+	}
+
+	// Make sure the final innerPath doesn't go outside the cgroup mountpoint.
+	if strings.HasPrefix(data.innerPath, "..") {
+		t.Errorf("SECURITY: cgroup innerPath is outside cgroup mountpoint!")
+	}
+
+	// Double-check, using an actual cgroup.
+	deviceRoot := filepath.Join(root, "devices")
+	devicePath, err := data.path("devices")
+	if err != nil {
+		t.Errorf("couldn't get cgroup path: %v", err)
+	}
+	if !strings.HasPrefix(devicePath, deviceRoot) {
+		t.Errorf("SECURITY: cgroup path() is outside cgroup mountpoint!")
+	}
+}
+
+// XXX: Remove me after we get rid of configs.Cgroup.Name and configs.Cgroup.Parent.
+func TestInvalidCgroupNameAndParent(t *testing.T) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		t.Errorf("couldn't get cgroup root: %v", err)
+	}
+
+	config := &configs.Cgroup{
+		Parent: "../../../../../../../../../../some/path",
+		Name:   "../../../../../../../../../../some/path",
+	}
+
+	data, err := getCgroupData(config, 0)
+	if err != nil {
+		t.Errorf("couldn't get cgroup data: %v", err)
+	}
+
+	// Make sure the final innerPath doesn't go outside the cgroup mountpoint.
+	if strings.HasPrefix(data.innerPath, "..") {
+		t.Errorf("SECURITY: cgroup innerPath is outside cgroup mountpoint!")
+	}
+
+	// Double-check, using an actual cgroup.
+	deviceRoot := filepath.Join(root, "devices")
+	devicePath, err := data.path("devices")
+	if err != nil {
+		t.Errorf("couldn't get cgroup path: %v", err)
+	}
+	if !strings.HasPrefix(devicePath, deviceRoot) {
+		t.Errorf("SECURITY: cgroup path() is outside cgroup mountpoint!")
+	}
+}
+
+// XXX: Remove me after we get rid of configs.Cgroup.Name and configs.Cgroup.Parent.
+func TestInvalidAbsoluteCgroupNameAndParent(t *testing.T) {
+	root, err := getCgroupRoot()
+	if err != nil {
+		t.Errorf("couldn't get cgroup root: %v", err)
+	}
+
+	config := &configs.Cgroup{
+		Parent: "/../../../../../../../../../../some/path",
+		Name:   "/../../../../../../../../../../some/path",
+	}
+
+	data, err := getCgroupData(config, 0)
+	if err != nil {
+		t.Errorf("couldn't get cgroup data: %v", err)
+	}
+
+	// Make sure the final innerPath doesn't go outside the cgroup mountpoint.
+	if strings.HasPrefix(data.innerPath, "..") {
+		t.Errorf("SECURITY: cgroup innerPath is outside cgroup mountpoint!")
+	}
+
+	// Double-check, using an actual cgroup.
+	deviceRoot := filepath.Join(root, "devices")
+	devicePath, err := data.path("devices")
+	if err != nil {
+		t.Errorf("couldn't get cgroup path: %v", err)
+	}
+	if !strings.HasPrefix(devicePath, deviceRoot) {
+		t.Errorf("SECURITY: cgroup path() is outside cgroup mountpoint!")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go

@@ -0,0 +1,237 @@
+// +build linux
+
+package fs
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type BlkioGroup struct {
+}
+
+func (s *BlkioGroup) Name() string {
+	return "blkio"
+}
+
+func (s *BlkioGroup) Apply(d *cgroupData) error {
+	_, err := d.join("blkio")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *BlkioGroup) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.BlkioWeight != 0 {
+		if err := writeFile(path, "blkio.weight", strconv.FormatUint(uint64(cgroup.Resources.BlkioWeight), 10)); err != nil {
+			return err
+		}
+	}
+
+	if cgroup.Resources.BlkioLeafWeight != 0 {
+		if err := writeFile(path, "blkio.leaf_weight", strconv.FormatUint(uint64(cgroup.Resources.BlkioLeafWeight), 10)); err != nil {
+			return err
+		}
+	}
+	for _, wd := range cgroup.Resources.BlkioWeightDevice {
+		if err := writeFile(path, "blkio.weight_device", wd.WeightString()); err != nil {
+			return err
+		}
+		if err := writeFile(path, "blkio.leaf_weight_device", wd.LeafWeightString()); err != nil {
+			return err
+		}
+	}
+	for _, td := range cgroup.Resources.BlkioThrottleReadBpsDevice {
+		if err := writeFile(path, "blkio.throttle.read_bps_device", td.String()); err != nil {
+			return err
+		}
+	}
+	for _, td := range cgroup.Resources.BlkioThrottleWriteBpsDevice {
+		if err := writeFile(path, "blkio.throttle.write_bps_device", td.String()); err != nil {
+			return err
+		}
+	}
+	for _, td := range cgroup.Resources.BlkioThrottleReadIOPSDevice {
+		if err := writeFile(path, "blkio.throttle.read_iops_device", td.String()); err != nil {
+			return err
+		}
+	}
+	for _, td := range cgroup.Resources.BlkioThrottleWriteIOPSDevice {
+		if err := writeFile(path, "blkio.throttle.write_iops_device", td.String()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *BlkioGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("blkio"))
+}
+
+/*
+examples:
+
+    blkio.sectors
+    8:0 6792
+
+    blkio.io_service_bytes
+    8:0 Read 1282048
+    8:0 Write 2195456
+    8:0 Sync 2195456
+    8:0 Async 1282048
+    8:0 Total 3477504
+    Total 3477504
+
+    blkio.io_serviced
+    8:0 Read 124
+    8:0 Write 104
+    8:0 Sync 104
+    8:0 Async 124
+    8:0 Total 228
+    Total 228
+
+    blkio.io_queued
+    8:0 Read 0
+    8:0 Write 0
+    8:0 Sync 0
+    8:0 Async 0
+    8:0 Total 0
+    Total 0
+*/
+
+func splitBlkioStatLine(r rune) bool {
+	return r == ' ' || r == ':'
+}
+
+func getBlkioStat(path string) ([]cgroups.BlkioStatEntry, error) {
+	var blkioStats []cgroups.BlkioStatEntry
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return blkioStats, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		// format: dev type amount
+		fields := strings.FieldsFunc(sc.Text(), splitBlkioStatLine)
+		if len(fields) < 3 {
+			if len(fields) == 2 && fields[0] == "Total" {
+				// skip total line
+				continue
+			} else {
+				return nil, fmt.Errorf("Invalid line found while parsing %s: %s", path, sc.Text())
+			}
+		}
+
+		v, err := strconv.ParseUint(fields[0], 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		major := v
+
+		v, err = strconv.ParseUint(fields[1], 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		minor := v
+
+		op := ""
+		valueField := 2
+		if len(fields) == 4 {
+			op = fields[2]
+			valueField = 3
+		}
+		v, err = strconv.ParseUint(fields[valueField], 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		blkioStats = append(blkioStats, cgroups.BlkioStatEntry{Major: major, Minor: minor, Op: op, Value: v})
+	}
+
+	return blkioStats, nil
+}
+
+func (s *BlkioGroup) GetStats(path string, stats *cgroups.Stats) error {
+	// Try to read CFQ stats available on all CFQ enabled kernels first
+	if blkioStats, err := getBlkioStat(filepath.Join(path, "blkio.io_serviced_recursive")); err == nil && blkioStats != nil {
+		return getCFQStats(path, stats)
+	}
+	return getStats(path, stats) // Use generic stats as fallback
+}
+
+func getCFQStats(path string, stats *cgroups.Stats) error {
+	var blkioStats []cgroups.BlkioStatEntry
+	var err error
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.sectors_recursive")); err != nil {
+		return err
+	}
+	stats.BlkioStats.SectorsRecursive = blkioStats
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_service_bytes_recursive")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoServiceBytesRecursive = blkioStats
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_serviced_recursive")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoServicedRecursive = blkioStats
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_queued_recursive")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoQueuedRecursive = blkioStats
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_service_time_recursive")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoServiceTimeRecursive = blkioStats
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_wait_time_recursive")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoWaitTimeRecursive = blkioStats
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.io_merged_recursive")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoMergedRecursive = blkioStats
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.time_recursive")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoTimeRecursive = blkioStats
+
+	return nil
+}
+
+func getStats(path string, stats *cgroups.Stats) error {
+	var blkioStats []cgroups.BlkioStatEntry
+	var err error
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.throttle.io_service_bytes")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoServiceBytesRecursive = blkioStats
+
+	if blkioStats, err = getBlkioStat(filepath.Join(path, "blkio.throttle.io_serviced")); err != nil {
+		return err
+	}
+	stats.BlkioStats.IoServicedRecursive = blkioStats
+
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio_test.go

@@ -0,0 +1,636 @@
+// +build linux
+
+package fs
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+const (
+	sectorsRecursiveContents      = `8:0 1024`
+	serviceBytesRecursiveContents = `8:0 Read 100
+8:0 Write 200
+8:0 Sync 300
+8:0 Async 500
+8:0 Total 500
+Total 500`
+	servicedRecursiveContents = `8:0 Read 10
+8:0 Write 40
+8:0 Sync 20
+8:0 Async 30
+8:0 Total 50
+Total 50`
+	queuedRecursiveContents = `8:0 Read 1
+8:0 Write 4
+8:0 Sync 2
+8:0 Async 3
+8:0 Total 5
+Total 5`
+	serviceTimeRecursiveContents = `8:0 Read 173959
+8:0 Write 0
+8:0 Sync 0
+8:0 Async 173959
+8:0 Total 17395
+Total 17395`
+	waitTimeRecursiveContents = `8:0 Read 15571
+8:0 Write 0
+8:0 Sync 0
+8:0 Async 15571
+8:0 Total 15571`
+	mergedRecursiveContents = `8:0 Read 5
+8:0 Write 10
+8:0 Sync 0
+8:0 Async 0
+8:0 Total 15
+Total 15`
+	timeRecursiveContents = `8:0 8`
+	throttleServiceBytes  = `8:0 Read 11030528
+8:0 Write 23
+8:0 Sync 42
+8:0 Async 11030528
+8:0 Total 11030528
+252:0 Read 11030528
+252:0 Write 23
+252:0 Sync 42
+252:0 Async 11030528
+252:0 Total 11030528
+Total 22061056`
+	throttleServiced = `8:0 Read 164
+8:0 Write 23
+8:0 Sync 42
+8:0 Async 164
+8:0 Total 164
+252:0 Read 164
+252:0 Write 23
+252:0 Sync 42
+252:0 Async 164
+252:0 Total 164
+Total 328`
+)
+
+func appendBlkioStatEntry(blkioStatEntries *[]cgroups.BlkioStatEntry, major, minor, value uint64, op string) {
+	*blkioStatEntries = append(*blkioStatEntries, cgroups.BlkioStatEntry{Major: major, Minor: minor, Value: value, Op: op})
+}
+
+func TestBlkioSetWeight(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+
+	const (
+		weightBefore = 100
+		weightAfter  = 200
+	)
+
+	helper.writeFileContents(map[string]string{
+		"blkio.weight": strconv.Itoa(weightBefore),
+	})
+
+	helper.CgroupData.config.Resources.BlkioWeight = weightAfter
+	blkio := &BlkioGroup{}
+	if err := blkio.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, "blkio.weight")
+	if err != nil {
+		t.Fatalf("Failed to parse blkio.weight - %s", err)
+	}
+
+	if value != weightAfter {
+		t.Fatal("Got the wrong value, set blkio.weight failed.")
+	}
+}
+
+func TestBlkioSetWeightDevice(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+
+	const (
+		weightDeviceBefore = "8:0 400"
+	)
+
+	wd := configs.NewWeightDevice(8, 0, 500, 0)
+	weightDeviceAfter := wd.WeightString()
+
+	helper.writeFileContents(map[string]string{
+		"blkio.weight_device": weightDeviceBefore,
+	})
+
+	helper.CgroupData.config.Resources.BlkioWeightDevice = []*configs.WeightDevice{wd}
+	blkio := &BlkioGroup{}
+	if err := blkio.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "blkio.weight_device")
+	if err != nil {
+		t.Fatalf("Failed to parse blkio.weight_device - %s", err)
+	}
+
+	if value != weightDeviceAfter {
+		t.Fatal("Got the wrong value, set blkio.weight_device failed.")
+	}
+}
+
+// regression #274
+func TestBlkioSetMultipleWeightDevice(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+
+	const (
+		weightDeviceBefore = "8:0 400"
+	)
+
+	wd1 := configs.NewWeightDevice(8, 0, 500, 0)
+	wd2 := configs.NewWeightDevice(8, 16, 500, 0)
+	// we cannot actually set and check both because normal ioutil.WriteFile
+	// when writing to cgroup file will overwrite the whole file content instead
+	// of updating it as the kernel is doing. Just check the second device
+	// is present will suffice for the test to ensure multiple writes are done.
+	weightDeviceAfter := wd2.WeightString()
+
+	helper.writeFileContents(map[string]string{
+		"blkio.weight_device": weightDeviceBefore,
+	})
+
+	helper.CgroupData.config.Resources.BlkioWeightDevice = []*configs.WeightDevice{wd1, wd2}
+	blkio := &BlkioGroup{}
+	if err := blkio.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "blkio.weight_device")
+	if err != nil {
+		t.Fatalf("Failed to parse blkio.weight_device - %s", err)
+	}
+
+	if value != weightDeviceAfter {
+		t.Fatal("Got the wrong value, set blkio.weight_device failed.")
+	}
+}
+
+func TestBlkioStats(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify expected stats.
+	expectedStats := cgroups.BlkioStats{}
+	appendBlkioStatEntry(&expectedStats.SectorsRecursive, 8, 0, 1024, "")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 100, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 200, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 300, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 500, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 500, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 10, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 40, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 20, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 30, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 50, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 1, "Read")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 4, "Write")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 2, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 3, "Async")
+	appendBlkioStatEntry(&expectedStats.IoQueuedRecursive, 8, 0, 5, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 173959, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceTimeRecursive, 8, 0, 17395, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Read")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Write")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Async")
+	appendBlkioStatEntry(&expectedStats.IoWaitTimeRecursive, 8, 0, 15571, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 5, "Read")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 10, "Write")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 0, "Async")
+	appendBlkioStatEntry(&expectedStats.IoMergedRecursive, 8, 0, 15, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoTimeRecursive, 8, 0, 8, "")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
+}
+
+func TestBlkioStatsNoSectorsFile(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatalf("Failed unexpectedly: %s", err)
+	}
+}
+
+func TestBlkioStatsNoServiceBytesFile(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_serviced_recursive":     servicedRecursiveContents,
+		"blkio.io_queued_recursive":       queuedRecursiveContents,
+		"blkio.sectors_recursive":         sectorsRecursiveContents,
+		"blkio.io_service_time_recursive": serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":    waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":       mergedRecursiveContents,
+		"blkio.time_recursive":            timeRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatalf("Failed unexpectedly: %s", err)
+	}
+}
+
+func TestBlkioStatsNoServicedFile(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatalf("Failed unexpectedly: %s", err)
+	}
+}
+
+func TestBlkioStatsNoQueuedFile(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatalf("Failed unexpectedly: %s", err)
+	}
+}
+
+func TestBlkioStatsNoServiceTimeFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatalf("Failed unexpectedly: %s", err)
+	}
+}
+
+func TestBlkioStatsNoWaitTimeFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatalf("Failed unexpectedly: %s", err)
+	}
+}
+
+func TestBlkioStatsNoMergedFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatalf("Failed unexpectedly: %s", err)
+	}
+}
+
+func TestBlkioStatsNoTimeFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": serviceBytesRecursiveContents,
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatalf("Failed unexpectedly: %s", err)
+	}
+}
+
+func TestBlkioStatsUnexpectedNumberOfFields(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": "8:0 Read 100 100",
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected to fail, but did not")
+	}
+}
+
+func TestBlkioStatsUnexpectedFieldType(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": "8:0 Read Write",
+		"blkio.io_serviced_recursive":      servicedRecursiveContents,
+		"blkio.io_queued_recursive":        queuedRecursiveContents,
+		"blkio.sectors_recursive":          sectorsRecursiveContents,
+		"blkio.io_service_time_recursive":  serviceTimeRecursiveContents,
+		"blkio.io_wait_time_recursive":     waitTimeRecursiveContents,
+		"blkio.io_merged_recursive":        mergedRecursiveContents,
+		"blkio.time_recursive":             timeRecursiveContents,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected to fail, but did not")
+	}
+}
+
+func TestNonCFQBlkioStats(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"blkio.io_service_bytes_recursive": "",
+		"blkio.io_serviced_recursive":      "",
+		"blkio.io_queued_recursive":        "",
+		"blkio.sectors_recursive":          "",
+		"blkio.io_service_time_recursive":  "",
+		"blkio.io_wait_time_recursive":     "",
+		"blkio.io_merged_recursive":        "",
+		"blkio.time_recursive":             "",
+		"blkio.throttle.io_service_bytes":  throttleServiceBytes,
+		"blkio.throttle.io_serviced":       throttleServiced,
+	})
+
+	blkio := &BlkioGroup{}
+	actualStats := *cgroups.NewStats()
+	err := blkio.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify expected stats.
+	expectedStats := cgroups.BlkioStats{}
+
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 11030528, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 23, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 42, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 11030528, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 8, 0, 11030528, "Total")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 11030528, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 23, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 42, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 11030528, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServiceBytesRecursive, 252, 0, 11030528, "Total")
+
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 164, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 23, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 42, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 164, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 8, 0, 164, "Total")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 164, "Read")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 23, "Write")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 42, "Sync")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 164, "Async")
+	appendBlkioStatEntry(&expectedStats.IoServicedRecursive, 252, 0, 164, "Total")
+
+	expectBlkioStatsEquals(t, expectedStats, actualStats.BlkioStats)
+}
+
+func TestBlkioSetThrottleReadBpsDevice(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+
+	const (
+		throttleBefore = `8:0 1024`
+	)
+
+	td := configs.NewThrottleDevice(8, 0, 2048)
+	throttleAfter := td.String()
+
+	helper.writeFileContents(map[string]string{
+		"blkio.throttle.read_bps_device": throttleBefore,
+	})
+
+	helper.CgroupData.config.Resources.BlkioThrottleReadBpsDevice = []*configs.ThrottleDevice{td}
+	blkio := &BlkioGroup{}
+	if err := blkio.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "blkio.throttle.read_bps_device")
+	if err != nil {
+		t.Fatalf("Failed to parse blkio.throttle.read_bps_device - %s", err)
+	}
+
+	if value != throttleAfter {
+		t.Fatal("Got the wrong value, set blkio.throttle.read_bps_device failed.")
+	}
+}
+func TestBlkioSetThrottleWriteBpsDevice(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+
+	const (
+		throttleBefore = `8:0 1024`
+	)
+
+	td := configs.NewThrottleDevice(8, 0, 2048)
+	throttleAfter := td.String()
+
+	helper.writeFileContents(map[string]string{
+		"blkio.throttle.write_bps_device": throttleBefore,
+	})
+
+	helper.CgroupData.config.Resources.BlkioThrottleWriteBpsDevice = []*configs.ThrottleDevice{td}
+	blkio := &BlkioGroup{}
+	if err := blkio.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "blkio.throttle.write_bps_device")
+	if err != nil {
+		t.Fatalf("Failed to parse blkio.throttle.write_bps_device - %s", err)
+	}
+
+	if value != throttleAfter {
+		t.Fatal("Got the wrong value, set blkio.throttle.write_bps_device failed.")
+	}
+}
+func TestBlkioSetThrottleReadIOpsDevice(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+
+	const (
+		throttleBefore = `8:0 1024`
+	)
+
+	td := configs.NewThrottleDevice(8, 0, 2048)
+	throttleAfter := td.String()
+
+	helper.writeFileContents(map[string]string{
+		"blkio.throttle.read_iops_device": throttleBefore,
+	})
+
+	helper.CgroupData.config.Resources.BlkioThrottleReadIOPSDevice = []*configs.ThrottleDevice{td}
+	blkio := &BlkioGroup{}
+	if err := blkio.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "blkio.throttle.read_iops_device")
+	if err != nil {
+		t.Fatalf("Failed to parse blkio.throttle.read_iops_device - %s", err)
+	}
+
+	if value != throttleAfter {
+		t.Fatal("Got the wrong value, set blkio.throttle.read_iops_device failed.")
+	}
+}
+func TestBlkioSetThrottleWriteIOpsDevice(t *testing.T) {
+	helper := NewCgroupTestUtil("blkio", t)
+	defer helper.cleanup()
+
+	const (
+		throttleBefore = `8:0 1024`
+	)
+
+	td := configs.NewThrottleDevice(8, 0, 2048)
+	throttleAfter := td.String()
+
+	helper.writeFileContents(map[string]string{
+		"blkio.throttle.write_iops_device": throttleBefore,
+	})
+
+	helper.CgroupData.config.Resources.BlkioThrottleWriteIOPSDevice = []*configs.ThrottleDevice{td}
+	blkio := &BlkioGroup{}
+	if err := blkio.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "blkio.throttle.write_iops_device")
+	if err != nil {
+		t.Fatalf("Failed to parse blkio.throttle.write_iops_device - %s", err)
+	}
+
+	if value != throttleAfter {
+		t.Fatal("Got the wrong value, set blkio.throttle.write_iops_device failed.")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpu.go

@@ -0,0 +1,94 @@
+// +build linux
+
+package fs
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type CpuGroup struct {
+}
+
+func (s *CpuGroup) Name() string {
+	return "cpu"
+}
+
+func (s *CpuGroup) Apply(d *cgroupData) error {
+	// We always want to join the cpu group, to allow fair cpu scheduling
+	// on a container basis
+	_, err := d.join("cpu")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *CpuGroup) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.CpuShares != 0 {
+		if err := writeFile(path, "cpu.shares", strconv.FormatInt(cgroup.Resources.CpuShares, 10)); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.CpuPeriod != 0 {
+		if err := writeFile(path, "cpu.cfs_period_us", strconv.FormatInt(cgroup.Resources.CpuPeriod, 10)); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.CpuQuota != 0 {
+		if err := writeFile(path, "cpu.cfs_quota_us", strconv.FormatInt(cgroup.Resources.CpuQuota, 10)); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.CpuRtPeriod != 0 {
+		if err := writeFile(path, "cpu.rt_period_us", strconv.FormatInt(cgroup.Resources.CpuRtPeriod, 10)); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.CpuRtRuntime != 0 {
+		if err := writeFile(path, "cpu.rt_runtime_us", strconv.FormatInt(cgroup.Resources.CpuRtRuntime, 10)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *CpuGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("cpu"))
+}
+
+func (s *CpuGroup) GetStats(path string, stats *cgroups.Stats) error {
+	f, err := os.Open(filepath.Join(path, "cpu.stat"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	defer f.Close()
+
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		t, v, err := getCgroupParamKeyValue(sc.Text())
+		if err != nil {
+			return err
+		}
+		switch t {
+		case "nr_periods":
+			stats.CpuStats.ThrottlingData.Periods = v
+
+		case "nr_throttled":
+			stats.CpuStats.ThrottlingData.ThrottledPeriods = v
+
+		case "throttled_time":
+			stats.CpuStats.ThrottlingData.ThrottledTime = v
+		}
+	}
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpu_test.go

@@ -0,0 +1,163 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+func TestCpuSetShares(t *testing.T) {
+	helper := NewCgroupTestUtil("cpu", t)
+	defer helper.cleanup()
+
+	const (
+		sharesBefore = 1024
+		sharesAfter  = 512
+	)
+
+	helper.writeFileContents(map[string]string{
+		"cpu.shares": strconv.Itoa(sharesBefore),
+	})
+
+	helper.CgroupData.config.Resources.CpuShares = sharesAfter
+	cpu := &CpuGroup{}
+	if err := cpu.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, "cpu.shares")
+	if err != nil {
+		t.Fatalf("Failed to parse cpu.shares - %s", err)
+	}
+
+	if value != sharesAfter {
+		t.Fatal("Got the wrong value, set cpu.shares failed.")
+	}
+}
+
+func TestCpuSetBandWidth(t *testing.T) {
+	helper := NewCgroupTestUtil("cpu", t)
+	defer helper.cleanup()
+
+	const (
+		quotaBefore     = 8000
+		quotaAfter      = 5000
+		periodBefore    = 10000
+		periodAfter     = 7000
+		rtRuntimeBefore = 8000
+		rtRuntimeAfter  = 5000
+		rtPeriodBefore  = 10000
+		rtPeriodAfter   = 7000
+	)
+
+	helper.writeFileContents(map[string]string{
+		"cpu.cfs_quota_us":  strconv.Itoa(quotaBefore),
+		"cpu.cfs_period_us": strconv.Itoa(periodBefore),
+		"cpu.rt_runtime_us": strconv.Itoa(rtRuntimeBefore),
+		"cpu.rt_period_us":  strconv.Itoa(rtPeriodBefore),
+	})
+
+	helper.CgroupData.config.Resources.CpuQuota = quotaAfter
+	helper.CgroupData.config.Resources.CpuPeriod = periodAfter
+	helper.CgroupData.config.Resources.CpuRtRuntime = rtRuntimeAfter
+	helper.CgroupData.config.Resources.CpuRtPeriod = rtPeriodAfter
+	cpu := &CpuGroup{}
+	if err := cpu.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	quota, err := getCgroupParamUint(helper.CgroupPath, "cpu.cfs_quota_us")
+	if err != nil {
+		t.Fatalf("Failed to parse cpu.cfs_quota_us - %s", err)
+	}
+	if quota != quotaAfter {
+		t.Fatal("Got the wrong value, set cpu.cfs_quota_us failed.")
+	}
+
+	period, err := getCgroupParamUint(helper.CgroupPath, "cpu.cfs_period_us")
+	if err != nil {
+		t.Fatalf("Failed to parse cpu.cfs_period_us - %s", err)
+	}
+	if period != periodAfter {
+		t.Fatal("Got the wrong value, set cpu.cfs_period_us failed.")
+	}
+	rtRuntime, err := getCgroupParamUint(helper.CgroupPath, "cpu.rt_runtime_us")
+	if err != nil {
+		t.Fatalf("Failed to parse cpu.rt_runtime_us - %s", err)
+	}
+	if rtRuntime != rtRuntimeAfter {
+		t.Fatal("Got the wrong value, set cpu.rt_runtime_us failed.")
+	}
+	rtPeriod, err := getCgroupParamUint(helper.CgroupPath, "cpu.rt_period_us")
+	if err != nil {
+		t.Fatalf("Failed to parse cpu.rt_period_us - %s", err)
+	}
+	if rtPeriod != rtPeriodAfter {
+		t.Fatal("Got the wrong value, set cpu.rt_period_us failed.")
+	}
+}
+
+func TestCpuStats(t *testing.T) {
+	helper := NewCgroupTestUtil("cpu", t)
+	defer helper.cleanup()
+
+	const (
+		kNrPeriods     = 2000
+		kNrThrottled   = 200
+		kThrottledTime = uint64(18446744073709551615)
+	)
+
+	cpuStatContent := fmt.Sprintf("nr_periods %d\n nr_throttled %d\n throttled_time %d\n",
+		kNrPeriods, kNrThrottled, kThrottledTime)
+	helper.writeFileContents(map[string]string{
+		"cpu.stat": cpuStatContent,
+	})
+
+	cpu := &CpuGroup{}
+	actualStats := *cgroups.NewStats()
+	err := cpu.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedStats := cgroups.ThrottlingData{
+		Periods:          kNrPeriods,
+		ThrottledPeriods: kNrThrottled,
+		ThrottledTime:    kThrottledTime}
+
+	expectThrottlingDataEquals(t, expectedStats, actualStats.CpuStats.ThrottlingData)
+}
+
+func TestNoCpuStatFile(t *testing.T) {
+	helper := NewCgroupTestUtil("cpu", t)
+	defer helper.cleanup()
+
+	cpu := &CpuGroup{}
+	actualStats := *cgroups.NewStats()
+	err := cpu.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal("Expected not to fail, but did")
+	}
+}
+
+func TestInvalidCpuStat(t *testing.T) {
+	helper := NewCgroupTestUtil("cpu", t)
+	defer helper.cleanup()
+	cpuStatContent := `nr_periods 2000
+	nr_throttled 200
+	throttled_time fortytwo`
+	helper.writeFileContents(map[string]string{
+		"cpu.stat": cpuStatContent,
+	})
+
+	cpu := &CpuGroup{}
+	actualStats := *cgroups.NewStats()
+	err := cpu.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failed stat parsing.")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuacct.go

@@ -0,0 +1,121 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/system"
+)
+
+const (
+	cgroupCpuacctStat   = "cpuacct.stat"
+	nanosecondsInSecond = 1000000000
+)
+
+var clockTicks = uint64(system.GetClockTicks())
+
+type CpuacctGroup struct {
+}
+
+func (s *CpuacctGroup) Name() string {
+	return "cpuacct"
+}
+
+func (s *CpuacctGroup) Apply(d *cgroupData) error {
+	// we just want to join this group even though we don't set anything
+	if _, err := d.join("cpuacct"); err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+
+	return nil
+}
+
+func (s *CpuacctGroup) Set(path string, cgroup *configs.Cgroup) error {
+	return nil
+}
+
+func (s *CpuacctGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("cpuacct"))
+}
+
+func (s *CpuacctGroup) GetStats(path string, stats *cgroups.Stats) error {
+	userModeUsage, kernelModeUsage, err := getCpuUsageBreakdown(path)
+	if err != nil {
+		return err
+	}
+
+	totalUsage, err := getCgroupParamUint(path, "cpuacct.usage")
+	if err != nil {
+		return err
+	}
+
+	percpuUsage, err := getPercpuUsage(path)
+	if err != nil {
+		return err
+	}
+
+	stats.CpuStats.CpuUsage.TotalUsage = totalUsage
+	stats.CpuStats.CpuUsage.PercpuUsage = percpuUsage
+	stats.CpuStats.CpuUsage.UsageInUsermode = userModeUsage
+	stats.CpuStats.CpuUsage.UsageInKernelmode = kernelModeUsage
+	return nil
+}
+
+// Returns user and kernel usage breakdown in nanoseconds.
+func getCpuUsageBreakdown(path string) (uint64, uint64, error) {
+	userModeUsage := uint64(0)
+	kernelModeUsage := uint64(0)
+	const (
+		userField   = "user"
+		systemField = "system"
+	)
+
+	// Expected format:
+	// user <usage in ticks>
+	// system <usage in ticks>
+	data, err := ioutil.ReadFile(filepath.Join(path, cgroupCpuacctStat))
+	if err != nil {
+		return 0, 0, err
+	}
+	fields := strings.Fields(string(data))
+	if len(fields) != 4 {
+		return 0, 0, fmt.Errorf("failure - %s is expected to have 4 fields", filepath.Join(path, cgroupCpuacctStat))
+	}
+	if fields[0] != userField {
+		return 0, 0, fmt.Errorf("unexpected field %q in %q, expected %q", fields[0], cgroupCpuacctStat, userField)
+	}
+	if fields[2] != systemField {
+		return 0, 0, fmt.Errorf("unexpected field %q in %q, expected %q", fields[2], cgroupCpuacctStat, systemField)
+	}
+	if userModeUsage, err = strconv.ParseUint(fields[1], 10, 64); err != nil {
+		return 0, 0, err
+	}
+	if kernelModeUsage, err = strconv.ParseUint(fields[3], 10, 64); err != nil {
+		return 0, 0, err
+	}
+
+	return (userModeUsage * nanosecondsInSecond) / clockTicks, (kernelModeUsage * nanosecondsInSecond) / clockTicks, nil
+}
+
+func getPercpuUsage(path string) ([]uint64, error) {
+	percpuUsage := []uint64{}
+	data, err := ioutil.ReadFile(filepath.Join(path, "cpuacct.usage_percpu"))
+	if err != nil {
+		return percpuUsage, err
+	}
+	for _, value := range strings.Fields(string(data)) {
+		value, err := strconv.ParseUint(value, 10, 64)
+		if err != nil {
+			return percpuUsage, fmt.Errorf("Unable to convert param value to uint64: %s", err)
+		}
+		percpuUsage = append(percpuUsage, value)
+	}
+	return percpuUsage, nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuset.go

@@ -0,0 +1,139 @@
+// +build linux
+
+package fs
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils"
+)
+
+type CpusetGroup struct {
+}
+
+func (s *CpusetGroup) Name() string {
+	return "cpuset"
+}
+
+func (s *CpusetGroup) Apply(d *cgroupData) error {
+	dir, err := d.path("cpuset")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return s.ApplyDir(dir, d.config, d.pid)
+}
+
+func (s *CpusetGroup) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.CpusetCpus != "" {
+		if err := writeFile(path, "cpuset.cpus", cgroup.Resources.CpusetCpus); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.CpusetMems != "" {
+		if err := writeFile(path, "cpuset.mems", cgroup.Resources.CpusetMems); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *CpusetGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("cpuset"))
+}
+
+func (s *CpusetGroup) GetStats(path string, stats *cgroups.Stats) error {
+	return nil
+}
+
+func (s *CpusetGroup) ApplyDir(dir string, cgroup *configs.Cgroup, pid int) error {
+	// This might happen if we have no cpuset cgroup mounted.
+	// Just do nothing and don't fail.
+	if dir == "" {
+		return nil
+	}
+	root, err := getCgroupRoot()
+	if err != nil {
+		return err
+	}
+	if err := s.ensureParent(dir, root); err != nil {
+		return err
+	}
+	// because we are not using d.join we need to place the pid into the procs file
+	// unlike the other subsystems
+	if err := writeFile(dir, "cgroup.procs", strconv.Itoa(pid)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *CpusetGroup) getSubsystemSettings(parent string) (cpus []byte, mems []byte, err error) {
+	if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus")); err != nil {
+		return
+	}
+	if mems, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.mems")); err != nil {
+		return
+	}
+	return cpus, mems, nil
+}
+
+// ensureParent makes sure that the parent directory of current is created
+// and populated with the proper cpus and mems files copied from
+// it's parent.
+func (s *CpusetGroup) ensureParent(current, root string) error {
+	parent := filepath.Dir(current)
+	if libcontainerUtils.CleanPath(parent) == root {
+		return nil
+	}
+	// Avoid infinite recursion.
+	if parent == current {
+		return fmt.Errorf("cpuset: cgroup parent path outside cgroup root")
+	}
+	if err := s.ensureParent(parent, root); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(current, 0755); err != nil {
+		return err
+	}
+	return s.copyIfNeeded(current, parent)
+}
+
+// copyIfNeeded copies the cpuset.cpus and cpuset.mems from the parent
+// directory to the current directory if the file's contents are 0
+func (s *CpusetGroup) copyIfNeeded(current, parent string) error {
+	var (
+		err                      error
+		currentCpus, currentMems []byte
+		parentCpus, parentMems   []byte
+	)
+
+	if currentCpus, currentMems, err = s.getSubsystemSettings(current); err != nil {
+		return err
+	}
+	if parentCpus, parentMems, err = s.getSubsystemSettings(parent); err != nil {
+		return err
+	}
+
+	if s.isEmpty(currentCpus) {
+		if err := writeFile(current, "cpuset.cpus", string(parentCpus)); err != nil {
+			return err
+		}
+	}
+	if s.isEmpty(currentMems) {
+		if err := writeFile(current, "cpuset.mems", string(parentMems)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *CpusetGroup) isEmpty(b []byte) bool {
+	return len(bytes.Trim(b, "\n")) == 0
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuset_test.go

@@ -0,0 +1,65 @@
+// +build linux
+
+package fs
+
+import (
+	"testing"
+)
+
+func TestCpusetSetCpus(t *testing.T) {
+	helper := NewCgroupTestUtil("cpuset", t)
+	defer helper.cleanup()
+
+	const (
+		cpusBefore = "0"
+		cpusAfter  = "1-3"
+	)
+
+	helper.writeFileContents(map[string]string{
+		"cpuset.cpus": cpusBefore,
+	})
+
+	helper.CgroupData.config.Resources.CpusetCpus = cpusAfter
+	cpuset := &CpusetGroup{}
+	if err := cpuset.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "cpuset.cpus")
+	if err != nil {
+		t.Fatalf("Failed to parse cpuset.cpus - %s", err)
+	}
+
+	if value != cpusAfter {
+		t.Fatal("Got the wrong value, set cpuset.cpus failed.")
+	}
+}
+
+func TestCpusetSetMems(t *testing.T) {
+	helper := NewCgroupTestUtil("cpuset", t)
+	defer helper.cleanup()
+
+	const (
+		memsBefore = "0"
+		memsAfter  = "1"
+	)
+
+	helper.writeFileContents(map[string]string{
+		"cpuset.mems": memsBefore,
+	})
+
+	helper.CgroupData.config.Resources.CpusetMems = memsAfter
+	cpuset := &CpusetGroup{}
+	if err := cpuset.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "cpuset.mems")
+	if err != nil {
+		t.Fatalf("Failed to parse cpuset.mems - %s", err)
+	}
+
+	if value != memsAfter {
+		t.Fatal("Got the wrong value, set cpuset.mems failed.")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/devices.go

@@ -0,0 +1,78 @@
+// +build linux
+
+package fs
+
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/system"
+)
+
+type DevicesGroup struct {
+}
+
+func (s *DevicesGroup) Name() string {
+	return "devices"
+}
+
+func (s *DevicesGroup) Apply(d *cgroupData) error {
+	_, err := d.join("devices")
+	if err != nil {
+		// We will return error even it's `not found` error, devices
+		// cgroup is hard requirement for container's security.
+		return err
+	}
+	return nil
+}
+
+func (s *DevicesGroup) Set(path string, cgroup *configs.Cgroup) error {
+	if system.RunningInUserNS() {
+		return nil
+	}
+
+	devices := cgroup.Resources.Devices
+	if len(devices) > 0 {
+		for _, dev := range devices {
+			file := "devices.deny"
+			if dev.Allow {
+				file = "devices.allow"
+			}
+			if err := writeFile(path, file, dev.CgroupString()); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	if !cgroup.Resources.AllowAllDevices {
+		if err := writeFile(path, "devices.deny", "a"); err != nil {
+			return err
+		}
+
+		for _, dev := range cgroup.Resources.AllowedDevices {
+			if err := writeFile(path, "devices.allow", dev.CgroupString()); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	if err := writeFile(path, "devices.allow", "a"); err != nil {
+		return err
+	}
+
+	for _, dev := range cgroup.Resources.DeniedDevices {
+		if err := writeFile(path, "devices.deny", dev.CgroupString()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *DevicesGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("devices"))
+}
+
+func (s *DevicesGroup) GetStats(path string, stats *cgroups.Stats) error {
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/devices_test.go

@@ -0,0 +1,84 @@
+// +build linux
+
+package fs
+
+import (
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+var (
+	allowedDevices = []*configs.Device{
+		{
+			Path:        "/dev/zero",
+			Type:        'c',
+			Major:       1,
+			Minor:       5,
+			Permissions: "rwm",
+			FileMode:    0666,
+		},
+	}
+	allowedList   = "c 1:5 rwm"
+	deniedDevices = []*configs.Device{
+		{
+			Path:        "/dev/null",
+			Type:        'c',
+			Major:       1,
+			Minor:       3,
+			Permissions: "rwm",
+			FileMode:    0666,
+		},
+	}
+	deniedList = "c 1:3 rwm"
+)
+
+func TestDevicesSetAllow(t *testing.T) {
+	helper := NewCgroupTestUtil("devices", t)
+	defer helper.cleanup()
+
+	helper.writeFileContents(map[string]string{
+		"devices.deny": "a",
+	})
+
+	helper.CgroupData.config.Resources.AllowAllDevices = false
+	helper.CgroupData.config.Resources.AllowedDevices = allowedDevices
+	devices := &DevicesGroup{}
+	if err := devices.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "devices.allow")
+	if err != nil {
+		t.Fatalf("Failed to parse devices.allow - %s", err)
+	}
+
+	if value != allowedList {
+		t.Fatal("Got the wrong value, set devices.allow failed.")
+	}
+}
+
+func TestDevicesSetDeny(t *testing.T) {
+	helper := NewCgroupTestUtil("devices", t)
+	defer helper.cleanup()
+
+	helper.writeFileContents(map[string]string{
+		"devices.allow": "a",
+	})
+
+	helper.CgroupData.config.Resources.AllowAllDevices = true
+	helper.CgroupData.config.Resources.DeniedDevices = deniedDevices
+	devices := &DevicesGroup{}
+	if err := devices.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "devices.deny")
+	if err != nil {
+		t.Fatalf("Failed to parse devices.deny - %s", err)
+	}
+
+	if value != deniedList {
+		t.Fatal("Got the wrong value, set devices.deny failed.")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/freezer.go

@@ -0,0 +1,61 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type FreezerGroup struct {
+}
+
+func (s *FreezerGroup) Name() string {
+	return "freezer"
+}
+
+func (s *FreezerGroup) Apply(d *cgroupData) error {
+	_, err := d.join("freezer")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *FreezerGroup) Set(path string, cgroup *configs.Cgroup) error {
+	switch cgroup.Resources.Freezer {
+	case configs.Frozen, configs.Thawed:
+		if err := writeFile(path, "freezer.state", string(cgroup.Resources.Freezer)); err != nil {
+			return err
+		}
+
+		for {
+			state, err := readFile(path, "freezer.state")
+			if err != nil {
+				return err
+			}
+			if strings.TrimSpace(state) == string(cgroup.Resources.Freezer) {
+				break
+			}
+			time.Sleep(1 * time.Millisecond)
+		}
+	case configs.Undefined:
+		return nil
+	default:
+		return fmt.Errorf("Invalid argument '%s' to freezer.state", string(cgroup.Resources.Freezer))
+	}
+
+	return nil
+}
+
+func (s *FreezerGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("freezer"))
+}
+
+func (s *FreezerGroup) GetStats(path string, stats *cgroups.Stats) error {
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/freezer_test.go

@@ -0,0 +1,47 @@
+// +build linux
+
+package fs
+
+import (
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func TestFreezerSetState(t *testing.T) {
+	helper := NewCgroupTestUtil("freezer", t)
+	defer helper.cleanup()
+
+	helper.writeFileContents(map[string]string{
+		"freezer.state": string(configs.Frozen),
+	})
+
+	helper.CgroupData.config.Resources.Freezer = configs.Thawed
+	freezer := &FreezerGroup{}
+	if err := freezer.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "freezer.state")
+	if err != nil {
+		t.Fatalf("Failed to parse freezer.state - %s", err)
+	}
+	if value != string(configs.Thawed) {
+		t.Fatal("Got the wrong value, set freezer.state failed.")
+	}
+}
+
+func TestFreezerSetInvalidState(t *testing.T) {
+	helper := NewCgroupTestUtil("freezer", t)
+	defer helper.cleanup()
+
+	const (
+		invalidArg configs.FreezerState = "Invalid"
+	)
+
+	helper.CgroupData.config.Resources.Freezer = invalidArg
+	freezer := &FreezerGroup{}
+	if err := freezer.Set(helper.CgroupPath, helper.CgroupData.config); err == nil {
+		t.Fatal("Failed to return invalid argument error")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/fs_unsupported.go

@@ -0,0 +1,3 @@
+// +build !linux
+
+package fs

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/hugetlb.go

@@ -0,0 +1,71 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type HugetlbGroup struct {
+}
+
+func (s *HugetlbGroup) Name() string {
+	return "hugetlb"
+}
+
+func (s *HugetlbGroup) Apply(d *cgroupData) error {
+	_, err := d.join("hugetlb")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *HugetlbGroup) Set(path string, cgroup *configs.Cgroup) error {
+	for _, hugetlb := range cgroup.Resources.HugetlbLimit {
+		if err := writeFile(path, strings.Join([]string{"hugetlb", hugetlb.Pagesize, "limit_in_bytes"}, "."), strconv.FormatUint(hugetlb.Limit, 10)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *HugetlbGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("hugetlb"))
+}
+
+func (s *HugetlbGroup) GetStats(path string, stats *cgroups.Stats) error {
+	hugetlbStats := cgroups.HugetlbStats{}
+	for _, pageSize := range HugePageSizes {
+		usage := strings.Join([]string{"hugetlb", pageSize, "usage_in_bytes"}, ".")
+		value, err := getCgroupParamUint(path, usage)
+		if err != nil {
+			return fmt.Errorf("failed to parse %s - %v", usage, err)
+		}
+		hugetlbStats.Usage = value
+
+		maxUsage := strings.Join([]string{"hugetlb", pageSize, "max_usage_in_bytes"}, ".")
+		value, err = getCgroupParamUint(path, maxUsage)
+		if err != nil {
+			return fmt.Errorf("failed to parse %s - %v", maxUsage, err)
+		}
+		hugetlbStats.MaxUsage = value
+
+		failcnt := strings.Join([]string{"hugetlb", pageSize, "failcnt"}, ".")
+		value, err = getCgroupParamUint(path, failcnt)
+		if err != nil {
+			return fmt.Errorf("failed to parse %s - %v", failcnt, err)
+		}
+		hugetlbStats.Failcnt = value
+
+		stats.HugetlbStats[pageSize] = hugetlbStats
+	}
+
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/hugetlb_test.go

@@ -0,0 +1,154 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+const (
+	hugetlbUsageContents    = "128\n"
+	hugetlbMaxUsageContents = "256\n"
+	hugetlbFailcnt          = "100\n"
+)
+
+var (
+	usage    = "hugetlb.%s.usage_in_bytes"
+	limit    = "hugetlb.%s.limit_in_bytes"
+	maxUsage = "hugetlb.%s.max_usage_in_bytes"
+	failcnt  = "hugetlb.%s.failcnt"
+)
+
+func TestHugetlbSetHugetlb(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+
+	const (
+		hugetlbBefore = 256
+		hugetlbAfter  = 512
+	)
+
+	for _, pageSize := range HugePageSizes {
+		helper.writeFileContents(map[string]string{
+			fmt.Sprintf(limit, pageSize): strconv.Itoa(hugetlbBefore),
+		})
+	}
+
+	for _, pageSize := range HugePageSizes {
+		helper.CgroupData.config.Resources.HugetlbLimit = []*configs.HugepageLimit{
+			{
+				Pagesize: pageSize,
+				Limit:    hugetlbAfter,
+			},
+		}
+		hugetlb := &HugetlbGroup{}
+		if err := hugetlb.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	for _, pageSize := range HugePageSizes {
+		limit := fmt.Sprintf(limit, pageSize)
+		value, err := getCgroupParamUint(helper.CgroupPath, limit)
+		if err != nil {
+			t.Fatalf("Failed to parse %s - %s", limit, err)
+		}
+		if value != hugetlbAfter {
+			t.Fatalf("Set hugetlb.limit_in_bytes failed. Expected: %v, Got: %v", hugetlbAfter, value)
+		}
+	}
+}
+
+func TestHugetlbStats(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	for _, pageSize := range HugePageSizes {
+		helper.writeFileContents(map[string]string{
+			fmt.Sprintf(usage, pageSize):    hugetlbUsageContents,
+			fmt.Sprintf(maxUsage, pageSize): hugetlbMaxUsageContents,
+			fmt.Sprintf(failcnt, pageSize):  hugetlbFailcnt,
+		})
+	}
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedStats := cgroups.HugetlbStats{Usage: 128, MaxUsage: 256, Failcnt: 100}
+	for _, pageSize := range HugePageSizes {
+		expectHugetlbStatEquals(t, expectedStats, actualStats.HugetlbStats[pageSize])
+	}
+}
+
+func TestHugetlbStatsNoUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		maxUsage: hugetlbMaxUsageContents,
+	})
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestHugetlbStatsNoMaxUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	for _, pageSize := range HugePageSizes {
+		helper.writeFileContents(map[string]string{
+			fmt.Sprintf(usage, pageSize): hugetlbUsageContents,
+		})
+	}
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestHugetlbStatsBadUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	for _, pageSize := range HugePageSizes {
+		helper.writeFileContents(map[string]string{
+			fmt.Sprintf(usage, pageSize): "bad",
+			maxUsage:                     hugetlbMaxUsageContents,
+		})
+	}
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestHugetlbStatsBadMaxUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("hugetlb", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		usage:    hugetlbUsageContents,
+		maxUsage: "bad",
+	})
+
+	hugetlb := &HugetlbGroup{}
+	actualStats := *cgroups.NewStats()
+	err := hugetlb.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/memory.go

@@ -0,0 +1,201 @@
+// +build linux
+
+package fs
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type MemoryGroup struct {
+}
+
+func (s *MemoryGroup) Name() string {
+	return "memory"
+}
+
+func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
+	path, err := d.path("memory")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	if memoryAssigned(d.config) {
+		if path != "" {
+			if err := os.MkdirAll(path, 0755); err != nil {
+				return err
+			}
+		}
+		// We have to set kernel memory here, as we can't change it once
+		// processes have been attached.
+		if err := s.SetKernelMemory(path, d.config); err != nil {
+			return err
+		}
+	}
+
+	defer func() {
+		if err != nil {
+			os.RemoveAll(path)
+		}
+	}()
+
+	// We need to join memory cgroup after set memory limits, because
+	// kmem.limit_in_bytes can only be set when the cgroup is empty.
+	_, err = d.join("memory")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *MemoryGroup) SetKernelMemory(path string, cgroup *configs.Cgroup) error {
+	// This has to be done separately because it has special constraints (it
+	// can't be done after there are processes attached to the cgroup).
+	if cgroup.Resources.KernelMemory > 0 {
+		if err := writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemory, 10)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.Memory != 0 {
+		if err := writeFile(path, "memory.limit_in_bytes", strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.MemoryReservation != 0 {
+		if err := writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.MemorySwap > 0 {
+		if err := writeFile(path, "memory.memsw.limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.OomKillDisable {
+		if err := writeFile(path, "memory.oom_control", "1"); err != nil {
+			return err
+		}
+	}
+	if cgroup.Resources.MemorySwappiness == nil || int64(*cgroup.Resources.MemorySwappiness) == -1 {
+		return nil
+	} else if int64(*cgroup.Resources.MemorySwappiness) >= 0 && int64(*cgroup.Resources.MemorySwappiness) <= 100 {
+		if err := writeFile(path, "memory.swappiness", strconv.FormatInt(*cgroup.Resources.MemorySwappiness, 10)); err != nil {
+			return err
+		}
+	} else {
+		return fmt.Errorf("invalid value:%d. valid memory swappiness range is 0-100", int64(*cgroup.Resources.MemorySwappiness))
+	}
+
+	return nil
+}
+
+func (s *MemoryGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("memory"))
+}
+
+func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
+	// Set stats from memory.stat.
+	statsFile, err := os.Open(filepath.Join(path, "memory.stat"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+	defer statsFile.Close()
+
+	sc := bufio.NewScanner(statsFile)
+	for sc.Scan() {
+		t, v, err := getCgroupParamKeyValue(sc.Text())
+		if err != nil {
+			return fmt.Errorf("failed to parse memory.stat (%q) - %v", sc.Text(), err)
+		}
+		stats.MemoryStats.Stats[t] = v
+	}
+	stats.MemoryStats.Cache = stats.MemoryStats.Stats["cache"]
+
+	memoryUsage, err := getMemoryData(path, "")
+	if err != nil {
+		return err
+	}
+	stats.MemoryStats.Usage = memoryUsage
+	swapUsage, err := getMemoryData(path, "memsw")
+	if err != nil {
+		return err
+	}
+	stats.MemoryStats.SwapUsage = swapUsage
+	kernelUsage, err := getMemoryData(path, "kmem")
+	if err != nil {
+		return err
+	}
+	stats.MemoryStats.KernelUsage = kernelUsage
+
+	return nil
+}
+
+func memoryAssigned(cgroup *configs.Cgroup) bool {
+	return cgroup.Resources.Memory != 0 ||
+		cgroup.Resources.MemoryReservation != 0 ||
+		cgroup.Resources.MemorySwap > 0 ||
+		cgroup.Resources.KernelMemory > 0 ||
+		cgroup.Resources.OomKillDisable ||
+		(cgroup.Resources.MemorySwappiness != nil && *cgroup.Resources.MemorySwappiness != -1)
+}
+
+func getMemoryData(path, name string) (cgroups.MemoryData, error) {
+	memoryData := cgroups.MemoryData{}
+
+	moduleName := "memory"
+	if name != "" {
+		moduleName = strings.Join([]string{"memory", name}, ".")
+	}
+	usage := strings.Join([]string{moduleName, "usage_in_bytes"}, ".")
+	maxUsage := strings.Join([]string{moduleName, "max_usage_in_bytes"}, ".")
+	failcnt := strings.Join([]string{moduleName, "failcnt"}, ".")
+	limit := strings.Join([]string{moduleName, "limit_in_bytes"}, ".")
+
+	value, err := getCgroupParamUint(path, usage)
+	if err != nil {
+		if moduleName != "memory" && os.IsNotExist(err) {
+			return cgroups.MemoryData{}, nil
+		}
+		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", usage, err)
+	}
+	memoryData.Usage = value
+	value, err = getCgroupParamUint(path, maxUsage)
+	if err != nil {
+		if moduleName != "memory" && os.IsNotExist(err) {
+			return cgroups.MemoryData{}, nil
+		}
+		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", maxUsage, err)
+	}
+	memoryData.MaxUsage = value
+	value, err = getCgroupParamUint(path, failcnt)
+	if err != nil {
+		if moduleName != "memory" && os.IsNotExist(err) {
+			return cgroups.MemoryData{}, nil
+		}
+		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", failcnt, err)
+	}
+	memoryData.Failcnt = value
+	value, err = getCgroupParamUint(path, limit)
+	if err != nil {
+		if moduleName != "memory" && os.IsNotExist(err) {
+			return cgroups.MemoryData{}, nil
+		}
+		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", limit, err)
+	}
+	memoryData.Limit = value
+
+	return memoryData, nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/memory_test.go

@@ -0,0 +1,337 @@
+// +build linux
+
+package fs
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+const (
+	memoryStatContents = `cache 512
+rss 1024`
+	memoryUsageContents    = "2048\n"
+	memoryMaxUsageContents = "4096\n"
+	memoryFailcnt          = "100\n"
+	memoryLimitContents    = "8192\n"
+)
+
+func TestMemorySetMemory(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+
+	const (
+		memoryBefore      = 314572800 // 300M
+		memoryAfter       = 524288000 // 500M
+		reservationBefore = 209715200 // 200M
+		reservationAfter  = 314572800 // 300M
+	)
+
+	helper.writeFileContents(map[string]string{
+		"memory.limit_in_bytes":      strconv.Itoa(memoryBefore),
+		"memory.soft_limit_in_bytes": strconv.Itoa(reservationBefore),
+	})
+
+	helper.CgroupData.config.Resources.Memory = memoryAfter
+	helper.CgroupData.config.Resources.MemoryReservation = reservationAfter
+	memory := &MemoryGroup{}
+	if err := memory.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, "memory.limit_in_bytes")
+	if err != nil {
+		t.Fatalf("Failed to parse memory.limit_in_bytes - %s", err)
+	}
+	if value != memoryAfter {
+		t.Fatal("Got the wrong value, set memory.limit_in_bytes failed.")
+	}
+
+	value, err = getCgroupParamUint(helper.CgroupPath, "memory.soft_limit_in_bytes")
+	if err != nil {
+		t.Fatalf("Failed to parse memory.soft_limit_in_bytes - %s", err)
+	}
+	if value != reservationAfter {
+		t.Fatal("Got the wrong value, set memory.soft_limit_in_bytes failed.")
+	}
+}
+
+func TestMemorySetMemoryswap(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+
+	const (
+		memoryswapBefore = 314572800 // 300M
+		memoryswapAfter  = 524288000 // 500M
+	)
+
+	helper.writeFileContents(map[string]string{
+		"memory.memsw.limit_in_bytes": strconv.Itoa(memoryswapBefore),
+	})
+
+	helper.CgroupData.config.Resources.MemorySwap = memoryswapAfter
+	memory := &MemoryGroup{}
+	if err := memory.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, "memory.memsw.limit_in_bytes")
+	if err != nil {
+		t.Fatalf("Failed to parse memory.memsw.limit_in_bytes - %s", err)
+	}
+	if value != memoryswapAfter {
+		t.Fatal("Got the wrong value, set memory.memsw.limit_in_bytes failed.")
+	}
+}
+
+func TestMemorySetKernelMemory(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+
+	const (
+		kernelMemoryBefore = 314572800 // 300M
+		kernelMemoryAfter  = 524288000 // 500M
+	)
+
+	helper.writeFileContents(map[string]string{
+		"memory.kmem.limit_in_bytes": strconv.Itoa(kernelMemoryBefore),
+	})
+
+	helper.CgroupData.config.Resources.KernelMemory = kernelMemoryAfter
+	memory := &MemoryGroup{}
+	if err := memory.SetKernelMemory(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, "memory.kmem.limit_in_bytes")
+	if err != nil {
+		t.Fatalf("Failed to parse memory.kmem.limit_in_bytes - %s", err)
+	}
+	if value != kernelMemoryAfter {
+		t.Fatal("Got the wrong value, set memory.kmem.limit_in_bytes failed.")
+	}
+}
+
+func TestMemorySetMemorySwappinessDefault(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+
+	swappinessBefore := 60 //default is 60
+	swappinessAfter := int64(0)
+
+	helper.writeFileContents(map[string]string{
+		"memory.swappiness": strconv.Itoa(swappinessBefore),
+	})
+
+	helper.CgroupData.config.Resources.MemorySwappiness = &swappinessAfter
+	memory := &MemoryGroup{}
+	if err := memory.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, "memory.swappiness")
+	if err != nil {
+		t.Fatalf("Failed to parse memory.swappiness - %s", err)
+	}
+	if int64(value) != swappinessAfter {
+		t.Fatalf("Got the wrong value (%d), set memory.swappiness = %d failed.", value, swappinessAfter)
+	}
+}
+
+func TestMemoryStats(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.stat":                     memoryStatContents,
+		"memory.usage_in_bytes":           memoryUsageContents,
+		"memory.limit_in_bytes":           memoryLimitContents,
+		"memory.max_usage_in_bytes":       memoryMaxUsageContents,
+		"memory.failcnt":                  memoryFailcnt,
+		"memory.memsw.usage_in_bytes":     memoryUsageContents,
+		"memory.memsw.max_usage_in_bytes": memoryMaxUsageContents,
+		"memory.memsw.failcnt":            memoryFailcnt,
+		"memory.memsw.limit_in_bytes":     memoryLimitContents,
+		"memory.kmem.usage_in_bytes":      memoryUsageContents,
+		"memory.kmem.max_usage_in_bytes":  memoryMaxUsageContents,
+		"memory.kmem.failcnt":             memoryFailcnt,
+		"memory.kmem.limit_in_bytes":      memoryLimitContents,
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedStats := cgroups.MemoryStats{Cache: 512, Usage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, SwapUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, KernelUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, Stats: map[string]uint64{"cache": 512, "rss": 1024}}
+	expectMemoryStatEquals(t, expectedStats, actualStats.MemoryStats)
+}
+
+func TestMemoryStatsNoStatFile(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.usage_in_bytes":     memoryUsageContents,
+		"memory.max_usage_in_bytes": memoryMaxUsageContents,
+		"memory.limit_in_bytes":     memoryLimitContents,
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestMemoryStatsNoUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.stat":               memoryStatContents,
+		"memory.max_usage_in_bytes": memoryMaxUsageContents,
+		"memory.limit_in_bytes":     memoryLimitContents,
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestMemoryStatsNoMaxUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.stat":           memoryStatContents,
+		"memory.usage_in_bytes": memoryUsageContents,
+		"memory.limit_in_bytes": memoryLimitContents,
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestMemoryStatsNoLimitInBytesFile(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.stat":               memoryStatContents,
+		"memory.usage_in_bytes":     memoryUsageContents,
+		"memory.max_usage_in_bytes": memoryMaxUsageContents,
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestMemoryStatsBadStatFile(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.stat":               "rss rss",
+		"memory.usage_in_bytes":     memoryUsageContents,
+		"memory.max_usage_in_bytes": memoryMaxUsageContents,
+		"memory.limit_in_bytes":     memoryLimitContents,
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestMemoryStatsBadUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.stat":               memoryStatContents,
+		"memory.usage_in_bytes":     "bad",
+		"memory.max_usage_in_bytes": memoryMaxUsageContents,
+		"memory.limit_in_bytes":     memoryLimitContents,
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestMemoryStatsBadMaxUsageFile(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.stat":               memoryStatContents,
+		"memory.usage_in_bytes":     memoryUsageContents,
+		"memory.max_usage_in_bytes": "bad",
+		"memory.limit_in_bytes":     memoryLimitContents,
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestMemoryStatsBadLimitInBytesFile(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+	helper.writeFileContents(map[string]string{
+		"memory.stat":               memoryStatContents,
+		"memory.usage_in_bytes":     memoryUsageContents,
+		"memory.max_usage_in_bytes": memoryMaxUsageContents,
+		"memory.limit_in_bytes":     "bad",
+	})
+
+	memory := &MemoryGroup{}
+	actualStats := *cgroups.NewStats()
+	err := memory.GetStats(helper.CgroupPath, &actualStats)
+	if err == nil {
+		t.Fatal("Expected failure")
+	}
+}
+
+func TestMemorySetOomControl(t *testing.T) {
+	helper := NewCgroupTestUtil("memory", t)
+	defer helper.cleanup()
+
+	const (
+		oom_kill_disable = 1 // disable oom killer, default is 0
+	)
+
+	helper.writeFileContents(map[string]string{
+		"memory.oom_control": strconv.Itoa(oom_kill_disable),
+	})
+
+	memory := &MemoryGroup{}
+	if err := memory.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, "memory.oom_control")
+	if err != nil {
+		t.Fatalf("Failed to parse memory.oom_control - %s", err)
+	}
+
+	if value != oom_kill_disable {
+		t.Fatalf("Got the wrong value, set memory.oom_control failed.")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/name.go

@@ -0,0 +1,40 @@
+// +build linux
+
+package fs
+
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type NameGroup struct {
+	GroupName string
+	Join      bool
+}
+
+func (s *NameGroup) Name() string {
+	return s.GroupName
+}
+
+func (s *NameGroup) Apply(d *cgroupData) error {
+	if s.Join {
+		// ignore errors if the named cgroup does not exist
+		d.join(s.GroupName)
+	}
+	return nil
+}
+
+func (s *NameGroup) Set(path string, cgroup *configs.Cgroup) error {
+	return nil
+}
+
+func (s *NameGroup) Remove(d *cgroupData) error {
+	if s.Join {
+		removePath(d.path(s.GroupName))
+	}
+	return nil
+}
+
+func (s *NameGroup) GetStats(path string, stats *cgroups.Stats) error {
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_cls.go

@@ -0,0 +1,41 @@
+// +build linux
+
+package fs
+
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type NetClsGroup struct {
+}
+
+func (s *NetClsGroup) Name() string {
+	return "net_cls"
+}
+
+func (s *NetClsGroup) Apply(d *cgroupData) error {
+	_, err := d.join("net_cls")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *NetClsGroup) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.NetClsClassid != "" {
+		if err := writeFile(path, "net_cls.classid", cgroup.Resources.NetClsClassid); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *NetClsGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("net_cls"))
+}
+
+func (s *NetClsGroup) GetStats(path string, stats *cgroups.Stats) error {
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_cls_test.go

@@ -0,0 +1,38 @@
+// +build linux
+
+package fs
+
+import (
+	"testing"
+)
+
+const (
+	classidBefore = "0x100002"
+	classidAfter  = "0x100001"
+)
+
+func TestNetClsSetClassid(t *testing.T) {
+	helper := NewCgroupTestUtil("net_cls", t)
+	defer helper.cleanup()
+
+	helper.writeFileContents(map[string]string{
+		"net_cls.classid": classidBefore,
+	})
+
+	helper.CgroupData.config.Resources.NetClsClassid = classidAfter
+	netcls := &NetClsGroup{}
+	if err := netcls.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	// As we are in mock environment, we can't get correct value of classid from
+	// net_cls.classid.
+	// So. we just judge if we successfully write classid into file
+	value, err := getCgroupParamString(helper.CgroupPath, "net_cls.classid")
+	if err != nil {
+		t.Fatalf("Failed to parse net_cls.classid - %s", err)
+	}
+	if value != classidAfter {
+		t.Fatal("Got the wrong value, set net_cls.classid failed.")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_prio.go

@@ -0,0 +1,41 @@
+// +build linux
+
+package fs
+
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type NetPrioGroup struct {
+}
+
+func (s *NetPrioGroup) Name() string {
+	return "net_prio"
+}
+
+func (s *NetPrioGroup) Apply(d *cgroupData) error {
+	_, err := d.join("net_prio")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *NetPrioGroup) Set(path string, cgroup *configs.Cgroup) error {
+	for _, prioMap := range cgroup.Resources.NetPrioIfpriomap {
+		if err := writeFile(path, "net_prio.ifpriomap", prioMap.CgroupString()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *NetPrioGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("net_prio"))
+}
+
+func (s *NetPrioGroup) GetStats(path string, stats *cgroups.Stats) error {
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_prio_test.go

@@ -0,0 +1,38 @@
+// +build linux
+
+package fs
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+var (
+	prioMap = []*configs.IfPrioMap{
+		{
+			Interface: "test",
+			Priority:  5,
+		},
+	}
+)
+
+func TestNetPrioSetIfPrio(t *testing.T) {
+	helper := NewCgroupTestUtil("net_prio", t)
+	defer helper.cleanup()
+
+	helper.CgroupData.config.Resources.NetPrioIfpriomap = prioMap
+	netPrio := &NetPrioGroup{}
+	if err := netPrio.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "net_prio.ifpriomap")
+	if err != nil {
+		t.Fatalf("Failed to parse net_prio.ifpriomap - %s", err)
+	}
+	if !strings.Contains(value, "test 5") {
+		t.Fatal("Got the wrong value, set net_prio.ifpriomap failed.")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/perf_event.go

@@ -0,0 +1,35 @@
+// +build linux
+
+package fs
+
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type PerfEventGroup struct {
+}
+
+func (s *PerfEventGroup) Name() string {
+	return "perf_event"
+}
+
+func (s *PerfEventGroup) Apply(d *cgroupData) error {
+	// we just want to join this group even though we don't set anything
+	if _, err := d.join("perf_event"); err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *PerfEventGroup) Set(path string, cgroup *configs.Cgroup) error {
+	return nil
+}
+
+func (s *PerfEventGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("perf_event"))
+}
+
+func (s *PerfEventGroup) GetStats(path string, stats *cgroups.Stats) error {
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/pids.go

@@ -0,0 +1,63 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type PidsGroup struct {
+}
+
+func (s *PidsGroup) Name() string {
+	return "pids"
+}
+
+func (s *PidsGroup) Apply(d *cgroupData) error {
+	_, err := d.join("pids")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *PidsGroup) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.PidsLimit != 0 {
+		// "max" is the fallback value.
+		limit := "max"
+
+		if cgroup.Resources.PidsLimit > 0 {
+			limit = strconv.FormatInt(cgroup.Resources.PidsLimit, 10)
+		}
+
+		if err := writeFile(path, "pids.max", limit); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *PidsGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("pids"))
+}
+
+func (s *PidsGroup) GetStats(path string, stats *cgroups.Stats) error {
+	current, err := getCgroupParamUint(path, "pids.current")
+	if err != nil {
+		return fmt.Errorf("failed to parse pids.current - %s", err)
+	}
+
+	max, err := getCgroupParamUint(path, "pids.max")
+	if err != nil {
+		return fmt.Errorf("failed to parse pids.max - %s", err)
+	}
+
+	stats.PidsStats.Current = current
+	stats.PidsStats.Max = max
+	return nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/pids_test.go

@@ -0,0 +1,87 @@
+// +build linux
+
+package fs
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+const (
+	maxUnlimited = -1
+	maxLimited   = 1024
+)
+
+func TestPidsSetMax(t *testing.T) {
+	helper := NewCgroupTestUtil("pids", t)
+	defer helper.cleanup()
+
+	helper.writeFileContents(map[string]string{
+		"pids.max": "max",
+	})
+
+	helper.CgroupData.config.Resources.PidsLimit = maxLimited
+	pids := &PidsGroup{}
+	if err := pids.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUint(helper.CgroupPath, "pids.max")
+	if err != nil {
+		t.Fatalf("Failed to parse pids.max - %s", err)
+	}
+
+	if value != maxLimited {
+		t.Fatalf("Expected %d, got %d for setting pids.max - limited", maxLimited, value)
+	}
+}
+
+func TestPidsSetUnlimited(t *testing.T) {
+	helper := NewCgroupTestUtil("pids", t)
+	defer helper.cleanup()
+
+	helper.writeFileContents(map[string]string{
+		"pids.max": strconv.Itoa(maxLimited),
+	})
+
+	helper.CgroupData.config.Resources.PidsLimit = maxUnlimited
+	pids := &PidsGroup{}
+	if err := pids.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamString(helper.CgroupPath, "pids.max")
+	if err != nil {
+		t.Fatalf("Failed to parse pids.max - %s", err)
+	}
+
+	if value != "max" {
+		t.Fatalf("Expected %s, got %s for setting pids.max - unlimited", "max", value)
+	}
+}
+
+func TestPidsStats(t *testing.T) {
+	helper := NewCgroupTestUtil("pids", t)
+	defer helper.cleanup()
+
+	helper.writeFileContents(map[string]string{
+		"pids.current": strconv.Itoa(1337),
+		"pids.max":     strconv.Itoa(maxLimited),
+	})
+
+	pids := &PidsGroup{}
+	stats := *cgroups.NewStats()
+	if err := pids.GetStats(helper.CgroupPath, &stats); err != nil {
+		t.Fatal(err)
+	}
+
+	if stats.PidsStats.Current != 1337 {
+		t.Fatalf("Expected %d, got %d for pids.current", 1337, stats.PidsStats.Current)
+	}
+
+	if stats.PidsStats.Max != maxLimited {
+		t.Fatalf("Expected %d, got %d for pids.max", maxLimited, stats.PidsStats.Max)
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/stats_util_test.go

@@ -0,0 +1,117 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+func blkioStatEntryEquals(expected, actual []cgroups.BlkioStatEntry) error {
+	if len(expected) != len(actual) {
+		return fmt.Errorf("blkioStatEntries length do not match")
+	}
+	for i, expValue := range expected {
+		actValue := actual[i]
+		if expValue != actValue {
+			return fmt.Errorf("Expected blkio stat entry %v but found %v", expValue, actValue)
+		}
+	}
+	return nil
+}
+
+func expectBlkioStatsEquals(t *testing.T, expected, actual cgroups.BlkioStats) {
+	if err := blkioStatEntryEquals(expected.IoServiceBytesRecursive, actual.IoServiceBytesRecursive); err != nil {
+		logrus.Printf("blkio IoServiceBytesRecursive do not match - %s\n", err)
+		t.Fail()
+	}
+
+	if err := blkioStatEntryEquals(expected.IoServicedRecursive, actual.IoServicedRecursive); err != nil {
+		logrus.Printf("blkio IoServicedRecursive do not match - %s\n", err)
+		t.Fail()
+	}
+
+	if err := blkioStatEntryEquals(expected.IoQueuedRecursive, actual.IoQueuedRecursive); err != nil {
+		logrus.Printf("blkio IoQueuedRecursive do not match - %s\n", err)
+		t.Fail()
+	}
+
+	if err := blkioStatEntryEquals(expected.SectorsRecursive, actual.SectorsRecursive); err != nil {
+		logrus.Printf("blkio SectorsRecursive do not match - %s\n", err)
+		t.Fail()
+	}
+
+	if err := blkioStatEntryEquals(expected.IoServiceTimeRecursive, actual.IoServiceTimeRecursive); err != nil {
+		logrus.Printf("blkio IoServiceTimeRecursive do not match - %s\n", err)
+		t.Fail()
+	}
+
+	if err := blkioStatEntryEquals(expected.IoWaitTimeRecursive, actual.IoWaitTimeRecursive); err != nil {
+		logrus.Printf("blkio IoWaitTimeRecursive do not match - %s\n", err)
+		t.Fail()
+	}
+
+	if err := blkioStatEntryEquals(expected.IoMergedRecursive, actual.IoMergedRecursive); err != nil {
+		logrus.Printf("blkio IoMergedRecursive do not match - %v vs %v\n", expected.IoMergedRecursive, actual.IoMergedRecursive)
+		t.Fail()
+	}
+
+	if err := blkioStatEntryEquals(expected.IoTimeRecursive, actual.IoTimeRecursive); err != nil {
+		logrus.Printf("blkio IoTimeRecursive do not match - %s\n", err)
+		t.Fail()
+	}
+}
+
+func expectThrottlingDataEquals(t *testing.T, expected, actual cgroups.ThrottlingData) {
+	if expected != actual {
+		logrus.Printf("Expected throttling data %v but found %v\n", expected, actual)
+		t.Fail()
+	}
+}
+
+func expectHugetlbStatEquals(t *testing.T, expected, actual cgroups.HugetlbStats) {
+	if expected != actual {
+		logrus.Printf("Expected hugetlb stats %v but found %v\n", expected, actual)
+		t.Fail()
+	}
+}
+
+func expectMemoryStatEquals(t *testing.T, expected, actual cgroups.MemoryStats) {
+	expectMemoryDataEquals(t, expected.Usage, actual.Usage)
+	expectMemoryDataEquals(t, expected.SwapUsage, actual.SwapUsage)
+	expectMemoryDataEquals(t, expected.KernelUsage, actual.KernelUsage)
+
+	for key, expValue := range expected.Stats {
+		actValue, ok := actual.Stats[key]
+		if !ok {
+			logrus.Printf("Expected memory stat key %s not found\n", key)
+			t.Fail()
+		}
+		if expValue != actValue {
+			logrus.Printf("Expected memory stat value %d but found %d\n", expValue, actValue)
+			t.Fail()
+		}
+	}
+}
+
+func expectMemoryDataEquals(t *testing.T, expected, actual cgroups.MemoryData) {
+	if expected.Usage != actual.Usage {
+		logrus.Printf("Expected memory usage %d but found %d\n", expected.Usage, actual.Usage)
+		t.Fail()
+	}
+	if expected.MaxUsage != actual.MaxUsage {
+		logrus.Printf("Expected memory max usage %d but found %d\n", expected.MaxUsage, actual.MaxUsage)
+		t.Fail()
+	}
+	if expected.Failcnt != actual.Failcnt {
+		logrus.Printf("Expected memory failcnt %d but found %d\n", expected.Failcnt, actual.Failcnt)
+		t.Fail()
+	}
+	if expected.Limit != actual.Limit {
+		logrus.Printf("Expected memory limit %d but found %d\n", expected.Limit, actual.Limit)
+		t.Fail()
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/util_test.go

@@ -0,0 +1,67 @@
+// +build linux
+
+/*
+Utility for testing cgroup operations.
+
+Creates a mock of the cgroup filesystem for the duration of the test.
+*/
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type cgroupTestUtil struct {
+	// cgroup data to use in tests.
+	CgroupData *cgroupData
+
+	// Path to the mock cgroup directory.
+	CgroupPath string
+
+	// Temporary directory to store mock cgroup filesystem.
+	tempDir string
+	t       *testing.T
+}
+
+// Creates a new test util for the specified subsystem
+func NewCgroupTestUtil(subsystem string, t *testing.T) *cgroupTestUtil {
+	d := &cgroupData{
+		config: &configs.Cgroup{},
+	}
+	d.config.Resources = &configs.Resources{}
+	tempDir, err := ioutil.TempDir("", "cgroup_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	d.root = tempDir
+	testCgroupPath := filepath.Join(d.root, subsystem)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure the full mock cgroup path exists.
+	err = os.MkdirAll(testCgroupPath, 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &cgroupTestUtil{CgroupData: d, CgroupPath: testCgroupPath, tempDir: tempDir, t: t}
+}
+
+func (c *cgroupTestUtil) cleanup() {
+	os.RemoveAll(c.tempDir)
+}
+
+// Write the specified contents on the mock of the specified cgroup files.
+func (c *cgroupTestUtil) writeFileContents(fileContents map[string]string) {
+	for file, contents := range fileContents {
+		err := writeFile(c.CgroupPath, file, contents)
+		if err != nil {
+			c.t.Fatal(err)
+		}
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/utils.go

@@ -0,0 +1,79 @@
+// +build linux
+
+package fs
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+var (
+	ErrNotSupportStat = errors.New("stats are not supported for subsystem")
+	ErrNotValidFormat = errors.New("line is not a valid key value format")
+)
+
+// Saturates negative values at zero and returns a uint64.
+// Due to kernel bugs, some of the memory cgroup stats can be negative.
+func parseUint(s string, base, bitSize int) (uint64, error) {
+	value, err := strconv.ParseUint(s, base, bitSize)
+	if err != nil {
+		intValue, intErr := strconv.ParseInt(s, base, bitSize)
+		// 1. Handle negative values greater than MinInt64 (and)
+		// 2. Handle negative values lesser than MinInt64
+		if intErr == nil && intValue < 0 {
+			return 0, nil
+		} else if intErr != nil && intErr.(*strconv.NumError).Err == strconv.ErrRange && intValue < 0 {
+			return 0, nil
+		}
+
+		return value, err
+	}
+
+	return value, nil
+}
+
+// Parses a cgroup param and returns as name, value
+//  i.e. "io_service_bytes 1234" will return as io_service_bytes, 1234
+func getCgroupParamKeyValue(t string) (string, uint64, error) {
+	parts := strings.Fields(t)
+	switch len(parts) {
+	case 2:
+		value, err := parseUint(parts[1], 10, 64)
+		if err != nil {
+			return "", 0, fmt.Errorf("unable to convert param value (%q) to uint64: %v", parts[1], err)
+		}
+
+		return parts[0], value, nil
+	default:
+		return "", 0, ErrNotValidFormat
+	}
+}
+
+// Gets a single uint64 value from the specified cgroup file.
+func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) {
+	fileName := filepath.Join(cgroupPath, cgroupFile)
+	contents, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		return 0, err
+	}
+
+	res, err := parseUint(strings.TrimSpace(string(contents)), 10, 64)
+	if err != nil {
+		return res, fmt.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), fileName)
+	}
+	return res, nil
+}
+
+// Gets a string value from the specified cgroup file
+func getCgroupParamString(cgroupPath, cgroupFile string) (string, error) {
+	contents, err := ioutil.ReadFile(filepath.Join(cgroupPath, cgroupFile))
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(contents)), nil
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/utils_test.go

@@ -0,0 +1,97 @@
+// +build linux
+
+package fs
+
+import (
+	"io/ioutil"
+	"math"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+)
+
+const (
+	cgroupFile  = "cgroup.file"
+	floatValue  = 2048.0
+	floatString = "2048"
+)
+
+func TestGetCgroupParamsInt(t *testing.T) {
+	// Setup tempdir.
+	tempDir, err := ioutil.TempDir("", "cgroup_utils_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempDir)
+	tempFile := filepath.Join(tempDir, cgroupFile)
+
+	// Success.
+	err = ioutil.WriteFile(tempFile, []byte(floatString), 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	value, err := getCgroupParamUint(tempDir, cgroupFile)
+	if err != nil {
+		t.Fatal(err)
+	} else if value != floatValue {
+		t.Fatalf("Expected %d to equal %f", value, floatValue)
+	}
+
+	// Success with new line.
+	err = ioutil.WriteFile(tempFile, []byte(floatString+"\n"), 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	value, err = getCgroupParamUint(tempDir, cgroupFile)
+	if err != nil {
+		t.Fatal(err)
+	} else if value != floatValue {
+		t.Fatalf("Expected %d to equal %f", value, floatValue)
+	}
+
+	// Success with negative values
+	err = ioutil.WriteFile(tempFile, []byte("-12345"), 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	value, err = getCgroupParamUint(tempDir, cgroupFile)
+	if err != nil {
+		t.Fatal(err)
+	} else if value != 0 {
+		t.Fatalf("Expected %d to equal %d", value, 0)
+	}
+
+	// Success with negative values lesser than min int64
+	s := strconv.FormatFloat(math.MinInt64, 'f', -1, 64)
+	err = ioutil.WriteFile(tempFile, []byte(s), 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	value, err = getCgroupParamUint(tempDir, cgroupFile)
+	if err != nil {
+		t.Fatal(err)
+	} else if value != 0 {
+		t.Fatalf("Expected %d to equal %d", value, 0)
+	}
+
+	// Not a float.
+	err = ioutil.WriteFile(tempFile, []byte("not-a-float"), 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = getCgroupParamUint(tempDir, cgroupFile)
+	if err == nil {
+		t.Fatal("Expecting error, got none")
+	}
+
+	// Unknown file.
+	err = os.Remove(tempFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = getCgroupParamUint(tempDir, cgroupFile)
+	if err == nil {
+		t.Fatal("Expecting error, got none")
+	}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/stats.go

@@ -0,0 +1,102 @@
+// +build linux
+
+package cgroups
+
+type ThrottlingData struct {
+	// Number of periods with throttling active
+	Periods uint64 `json:"periods,omitempty"`
+	// Number of periods when the container hit its throttling limit.
+	ThrottledPeriods uint64 `json:"throttled_periods,omitempty"`
+	// Aggregate time the container was throttled for in nanoseconds.
+	ThrottledTime uint64 `json:"throttled_time,omitempty"`
+}
+
+// All CPU stats are aggregate since container inception.
+type CpuUsage struct {
+	// Total CPU time consumed.
+	// Units: nanoseconds.
+	TotalUsage uint64 `json:"total_usage,omitempty"`
+	// Total CPU time consumed per core.
+	// Units: nanoseconds.
+	PercpuUsage []uint64 `json:"percpu_usage,omitempty"`
+	// Time spent by tasks of the cgroup in kernel mode.
+	// Units: nanoseconds.
+	UsageInKernelmode uint64 `json:"usage_in_kernelmode"`
+	// Time spent by tasks of the cgroup in user mode.
+	// Units: nanoseconds.
+	UsageInUsermode uint64 `json:"usage_in_usermode"`
+}
+
+type CpuStats struct {
+	CpuUsage       CpuUsage       `json:"cpu_usage,omitempty"`
+	ThrottlingData ThrottlingData `json:"throttling_data,omitempty"`
+}
+
+type MemoryData struct {
+	Usage    uint64 `json:"usage,omitempty"`
+	MaxUsage uint64 `json:"max_usage,omitempty"`
+	Failcnt  uint64 `json:"failcnt"`
+	Limit    uint64 `json:"limit"`
+}
+
+type MemoryStats struct {
+	// memory used for cache
+	Cache uint64 `json:"cache,omitempty"`
+	// usage of memory
+	Usage MemoryData `json:"usage,omitempty"`
+	// usage of memory + swap
+	SwapUsage MemoryData `json:"swap_usage,omitempty"`
+	// usage of kernel memory
+	KernelUsage MemoryData        `json:"kernel_usage,omitempty"`
+	Stats       map[string]uint64 `json:"stats,omitempty"`
+}
+
+type PidsStats struct {
+	// number of pids in the cgroup
+	Current uint64 `json:"current,omitempty"`
+	// active pids hard limit
+	Max uint64 `json:"max,omitempty"`
+}
+
+type BlkioStatEntry struct {
+	Major uint64 `json:"major,omitempty"`
+	Minor uint64 `json:"minor,omitempty"`
+	Op    string `json:"op,omitempty"`
+	Value uint64 `json:"value,omitempty"`
+}
+
+type BlkioStats struct {
+	// number of bytes tranferred to and from the block device
+	IoServiceBytesRecursive []BlkioStatEntry `json:"io_service_bytes_recursive,omitempty"`
+	IoServicedRecursive     []BlkioStatEntry `json:"io_serviced_recursive,omitempty"`
+	IoQueuedRecursive       []BlkioStatEntry `json:"io_queue_recursive,omitempty"`
+	IoServiceTimeRecursive  []BlkioStatEntry `json:"io_service_time_recursive,omitempty"`
+	IoWaitTimeRecursive     []BlkioStatEntry `json:"io_wait_time_recursive,omitempty"`
+	IoMergedRecursive       []BlkioStatEntry `json:"io_merged_recursive,omitempty"`
+	IoTimeRecursive         []BlkioStatEntry `json:"io_time_recursive,omitempty"`
+	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive,omitempty"`
+}
+
+type HugetlbStats struct {
+	// current res_counter usage for hugetlb
+	Usage uint64 `json:"usage,omitempty"`
+	// maximum usage ever recorded.
+	MaxUsage uint64 `json:"max_usage,omitempty"`
+	// number of times hugetlb usage allocation failure.
+	Failcnt uint64 `json:"failcnt"`
+}
+
+type Stats struct {
+	CpuStats    CpuStats    `json:"cpu_stats,omitempty"`
+	MemoryStats MemoryStats `json:"memory_stats,omitempty"`
+	PidsStats   PidsStats   `json:"pids_stats,omitempty"`
+	BlkioStats  BlkioStats  `json:"blkio_stats,omitempty"`
+	// the map is in the format "size of hugepage: stats of the hugepage"
+	HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`
+}
+
+func NewStats() *Stats {
+	memoryStats := MemoryStats{Stats: make(map[string]uint64)}
+	hugetlbStats := make(map[string]HugetlbStats)
+	return &Stats{MemoryStats: memoryStats, HugetlbStats: hugetlbStats}
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_nosystemd.go

@@ -0,0 +1,55 @@
+// +build !linux
+
+package systemd
+
+import (
+	"fmt"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type Manager struct {
+	Cgroups *configs.Cgroup
+	Paths   map[string]string
+}
+
+func UseSystemd() bool {
+	return false
+}
+
+func (m *Manager) Apply(pid int) error {
+	return fmt.Errorf("Systemd not supported")
+}
+
+func (m *Manager) GetPids() ([]int, error) {
+	return nil, fmt.Errorf("Systemd not supported")
+}
+
+func (m *Manager) GetAllPids() ([]int, error) {
+	return nil, fmt.Errorf("Systemd not supported")
+}
+
+func (m *Manager) Destroy() error {
+	return fmt.Errorf("Systemd not supported")
+}
+
+func (m *Manager) GetPaths() map[string]string {
+	return nil
+}
+
+func (m *Manager) GetStats() (*cgroups.Stats, error) {
+	return nil, fmt.Errorf("Systemd not supported")
+}
+
+func (m *Manager) Set(container *configs.Config) error {
+	return nil, fmt.Errorf("Systemd not supported")
+}
+
+func (m *Manager) Freeze(state configs.FreezerState) error {
+	return fmt.Errorf("Systemd not supported")
+}
+
+func Freeze(c *configs.Cgroup, state configs.FreezerState) error {
+	return fmt.Errorf("Systemd not supported")
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_systemd.go

@@ -0,0 +1,487 @@
+// +build linux
+
+package systemd
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	systemdDbus "github.com/coreos/go-systemd/dbus"
+	systemdUtil "github.com/coreos/go-systemd/util"
+	"github.com/godbus/dbus"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fs"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type Manager struct {
+	mu      sync.Mutex
+	Cgroups *configs.Cgroup
+	Paths   map[string]string
+}
+
+type subsystem interface {
+	// Name returns the name of the subsystem.
+	Name() string
+	// Returns the stats, as 'stats', corresponding to the cgroup under 'path'.
+	GetStats(path string, stats *cgroups.Stats) error
+	// Set the cgroup represented by cgroup.
+	Set(path string, cgroup *configs.Cgroup) error
+}
+
+var errSubsystemDoesNotExist = errors.New("cgroup: subsystem does not exist")
+
+type subsystemSet []subsystem
+
+func (s subsystemSet) Get(name string) (subsystem, error) {
+	for _, ss := range s {
+		if ss.Name() == name {
+			return ss, nil
+		}
+	}
+	return nil, errSubsystemDoesNotExist
+}
+
+var subsystems = subsystemSet{
+	&fs.CpusetGroup{},
+	&fs.DevicesGroup{},
+	&fs.MemoryGroup{},
+	&fs.CpuGroup{},
+	&fs.CpuacctGroup{},
+	&fs.PidsGroup{},
+	&fs.BlkioGroup{},
+	&fs.HugetlbGroup{},
+	&fs.PerfEventGroup{},
+	&fs.FreezerGroup{},
+	&fs.NetPrioGroup{},
+	&fs.NetClsGroup{},
+	&fs.NameGroup{GroupName: "name=systemd"},
+}
+
+const (
+	testScopeWait = 4
+)
+
+var (
+	connLock                        sync.Mutex
+	theConn                         *systemdDbus.Conn
+	hasStartTransientUnit           bool
+	hasTransientDefaultDependencies bool
+)
+
+func newProp(name string, units interface{}) systemdDbus.Property {
+	return systemdDbus.Property{
+		Name:  name,
+		Value: dbus.MakeVariant(units),
+	}
+}
+
+func UseSystemd() bool {
+	if !systemdUtil.IsRunningSystemd() {
+		return false
+	}
+
+	connLock.Lock()
+	defer connLock.Unlock()
+
+	if theConn == nil {
+		var err error
+		theConn, err = systemdDbus.New()
+		if err != nil {
+			return false
+		}
+
+		// Assume we have StartTransientUnit
+		hasStartTransientUnit = true
+
+		// But if we get UnknownMethod error we don't
+		if _, err := theConn.StartTransientUnit("test.scope", "invalid", nil, nil); err != nil {
+			if dbusError, ok := err.(dbus.Error); ok {
+				if dbusError.Name == "org.freedesktop.DBus.Error.UnknownMethod" {
+					hasStartTransientUnit = false
+					return hasStartTransientUnit
+				}
+			}
+		}
+
+		// Ensure the scope name we use doesn't exist. Use the Pid to
+		// avoid collisions between multiple libcontainer users on a
+		// single host.
+		scope := fmt.Sprintf("libcontainer-%d-systemd-test-default-dependencies.scope", os.Getpid())
+		testScopeExists := true
+		for i := 0; i <= testScopeWait; i++ {
+			if _, err := theConn.StopUnit(scope, "replace", nil); err != nil {
+				if dbusError, ok := err.(dbus.Error); ok {
+					if strings.Contains(dbusError.Name, "org.freedesktop.systemd1.NoSuchUnit") {
+						testScopeExists = false
+						break
+					}
+				}
+			}
+			time.Sleep(time.Millisecond)
+		}
+
+		// Bail out if we can't kill this scope without testing for DefaultDependencies
+		if testScopeExists {
+			return hasStartTransientUnit
+		}
+
+		// Assume StartTransientUnit on a scope allows DefaultDependencies
+		hasTransientDefaultDependencies = true
+		ddf := newProp("DefaultDependencies", false)
+		if _, err := theConn.StartTransientUnit(scope, "replace", []systemdDbus.Property{ddf}, nil); err != nil {
+			if dbusError, ok := err.(dbus.Error); ok {
+				if strings.Contains(dbusError.Name, "org.freedesktop.DBus.Error.PropertyReadOnly") {
+					hasTransientDefaultDependencies = false
+				}
+			}
+		}
+
+		// Not critical because of the stop unit logic above.
+		theConn.StopUnit(scope, "replace", nil)
+	}
+	return hasStartTransientUnit
+}
+
+func getIfaceForUnit(unitName string) string {
+	if strings.HasSuffix(unitName, ".scope") {
+		return "Scope"
+	}
+	if strings.HasSuffix(unitName, ".service") {
+		return "Service"
+	}
+	return "Unit"
+}
+
+func (m *Manager) Apply(pid int) error {
+	var (
+		c          = m.Cgroups
+		unitName   = getUnitName(c)
+		slice      = "system.slice"
+		properties []systemdDbus.Property
+	)
+
+	if c.Paths != nil {
+		paths := make(map[string]string)
+		for name, path := range c.Paths {
+			_, err := getSubsystemPath(m.Cgroups, name)
+			if err != nil {
+				// Don't fail if a cgroup hierarchy was not found, just skip this subsystem
+				if cgroups.IsNotFound(err) {
+					continue
+				}
+				return err
+			}
+			paths[name] = path
+		}
+		m.Paths = paths
+		return cgroups.EnterPid(m.Paths, pid)
+	}
+
+	if c.Parent != "" {
+		slice = c.Parent
+	}
+
+	properties = append(properties,
+		systemdDbus.PropSlice(slice),
+		systemdDbus.PropDescription("docker container "+c.Name),
+		newProp("PIDs", []uint32{uint32(pid)}),
+	)
+
+	// Always enable accounting, this gets us the same behaviour as the fs implementation,
+	// plus the kernel has some problems with joining the memory cgroup at a later time.
+	properties = append(properties,
+		newProp("MemoryAccounting", true),
+		newProp("CPUAccounting", true),
+		newProp("BlockIOAccounting", true))
+
+	if hasTransientDefaultDependencies {
+		properties = append(properties,
+			newProp("DefaultDependencies", false))
+	}
+
+	if c.Resources.Memory != 0 {
+		properties = append(properties,
+			newProp("MemoryLimit", uint64(c.Resources.Memory)))
+	}
+
+	if c.Resources.CpuShares != 0 {
+		properties = append(properties,
+			newProp("CPUShares", uint64(c.Resources.CpuShares)))
+	}
+
+	if c.Resources.BlkioWeight != 0 {
+		properties = append(properties,
+			newProp("BlockIOWeight", uint64(c.Resources.BlkioWeight)))
+	}
+
+	// We need to set kernel memory before processes join cgroup because
+	// kmem.limit_in_bytes can only be set when the cgroup is empty.
+	// And swap memory limit needs to be set after memory limit, only
+	// memory limit is handled by systemd, so it's kind of ugly here.
+	if c.Resources.KernelMemory > 0 {
+		if err := setKernelMemory(c); err != nil {
+			return err
+		}
+	}
+
+	if _, err := theConn.StartTransientUnit(unitName, "replace", properties, nil); err != nil {
+		return err
+	}
+
+	if err := joinCgroups(c, pid); err != nil {
+		return err
+	}
+
+	paths := make(map[string]string)
+	for _, s := range subsystems {
+		subsystemPath, err := getSubsystemPath(m.Cgroups, s.Name())
+		if err != nil {
+			// Don't fail if a cgroup hierarchy was not found, just skip this subsystem
+			if cgroups.IsNotFound(err) {
+				continue
+			}
+			return err
+		}
+		paths[s.Name()] = subsystemPath
+	}
+	m.Paths = paths
+	return nil
+}
+
+func (m *Manager) Destroy() error {
+	if m.Cgroups.Paths != nil {
+		return nil
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	theConn.StopUnit(getUnitName(m.Cgroups), "replace", nil)
+	if err := cgroups.RemovePaths(m.Paths); err != nil {
+		return err
+	}
+	m.Paths = make(map[string]string)
+	return nil
+}
+
+func (m *Manager) GetPaths() map[string]string {
+	m.mu.Lock()
+	paths := m.Paths
+	m.mu.Unlock()
+	return paths
+}
+
+func writeFile(dir, file, data string) error {
+	// Normally dir should not be empty, one case is that cgroup subsystem
+	// is not mounted, we will get empty dir, and we want it fail here.
+	if dir == "" {
+		return fmt.Errorf("no such directory for %s.", file)
+	}
+	return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700)
+}
+
+func join(c *configs.Cgroup, subsystem string, pid int) (string, error) {
+	path, err := getSubsystemPath(c, subsystem)
+	if err != nil {
+		return "", err
+	}
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return "", err
+	}
+	if err := writeFile(path, "cgroup.procs", strconv.Itoa(pid)); err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+func joinCgroups(c *configs.Cgroup, pid int) error {
+	for _, sys := range subsystems {
+		name := sys.Name()
+		switch name {
+		case "name=systemd":
+			// let systemd handle this
+			break
+		case "cpuset":
+			path, err := getSubsystemPath(c, name)
+			if err != nil && !cgroups.IsNotFound(err) {
+				return err
+			}
+			s := &fs.CpusetGroup{}
+			if err := s.ApplyDir(path, c, pid); err != nil {
+				return err
+			}
+			break
+		default:
+			_, err := join(c, name, pid)
+			if err != nil {
+				// Even if it's `not found` error, we'll return err
+				// because devices cgroup is hard requirement for
+				// container security.
+				if name == "devices" {
+					return err
+				}
+				// For other subsystems, omit the `not found` error
+				// because they are optional.
+				if !cgroups.IsNotFound(err) {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// systemd represents slice heirarchy using `-`, so we need to follow suit when
+// generating the path of slice. Essentially, test-a-b.slice becomes
+// test.slice/test-a.slice/test-a-b.slice.
+func expandSlice(slice string) (string, error) {
+	suffix := ".slice"
+	// Name has to end with ".slice", but can't be just ".slice".
+	if len(slice) < len(suffix) || !strings.HasSuffix(slice, suffix) {
+		return "", fmt.Errorf("invalid slice name: %s", slice)
+	}
+
+	// Path-separators are not allowed.
+	if strings.Contains(slice, "/") {
+		return "", fmt.Errorf("invalid slice name: %s", slice)
+	}
+
+	var path, prefix string
+	sliceName := strings.TrimSuffix(slice, suffix)
+	for _, component := range strings.Split(sliceName, "-") {
+		// test--a.slice isn't permitted, nor is -test.slice.
+		if component == "" {
+			return "", fmt.Errorf("invalid slice name: %s", slice)
+		}
+
+		// Append the component to the path and to the prefix.
+		path += prefix + component + suffix + "/"
+		prefix += component + "-"
+	}
+
+	return path, nil
+}
+
+func getSubsystemPath(c *configs.Cgroup, subsystem string) (string, error) {
+	mountpoint, err := cgroups.FindCgroupMountpoint(subsystem)
+	if err != nil {
+		return "", err
+	}
+
+	initPath, err := cgroups.GetInitCgroupDir(subsystem)
+	if err != nil {
+		return "", err
+	}
+
+	slice := "system.slice"
+	if c.Parent != "" {
+		slice = c.Parent
+	}
+
+	slice, err = expandSlice(slice)
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(mountpoint, initPath, slice, getUnitName(c)), nil
+}
+
+func (m *Manager) Freeze(state configs.FreezerState) error {
+	path, err := getSubsystemPath(m.Cgroups, "freezer")
+	if err != nil {
+		return err
+	}
+	prevState := m.Cgroups.Resources.Freezer
+	m.Cgroups.Resources.Freezer = state
+	freezer, err := subsystems.Get("freezer")
+	if err != nil {
+		return err
+	}
+	err = freezer.Set(path, m.Cgroups)
+	if err != nil {
+		m.Cgroups.Resources.Freezer = prevState
+		return err
+	}
+	return nil
+}
+
+func (m *Manager) GetPids() ([]int, error) {
+	path, err := getSubsystemPath(m.Cgroups, "devices")
+	if err != nil {
+		return nil, err
+	}
+	return cgroups.GetPids(path)
+}
+
+func (m *Manager) GetAllPids() ([]int, error) {
+	path, err := getSubsystemPath(m.Cgroups, "devices")
+	if err != nil {
+		return nil, err
+	}
+	return cgroups.GetAllPids(path)
+}
+
+func (m *Manager) GetStats() (*cgroups.Stats, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	stats := cgroups.NewStats()
+	for name, path := range m.Paths {
+		sys, err := subsystems.Get(name)
+		if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) {
+			continue
+		}
+		if err := sys.GetStats(path, stats); err != nil {
+			return nil, err
+		}
+	}
+
+	return stats, nil
+}
+
+func (m *Manager) Set(container *configs.Config) error {
+	for _, sys := range subsystems {
+		// Get the subsystem path, but don't error out for not found cgroups.
+		path, err := getSubsystemPath(container.Cgroups, sys.Name())
+		if err != nil && !cgroups.IsNotFound(err) {
+			return err
+		}
+
+		if err := sys.Set(path, container.Cgroups); err != nil {
+			return err
+		}
+	}
+
+	if m.Paths["cpu"] != "" {
+		if err := fs.CheckCpushares(m.Paths["cpu"], container.Cgroups.Resources.CpuShares); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getUnitName(c *configs.Cgroup) string {
+	return fmt.Sprintf("%s-%s.scope", c.ScopePrefix, c.Name)
+}
+
+func setKernelMemory(c *configs.Cgroup) error {
+	path, err := getSubsystemPath(c, "memory")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return err
+	}
+
+	// This doesn't get called by manager.Set, so we need to do it here.
+	s := &fs.MemoryGroup{}
+	return s.SetKernelMemory(path, c)
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go

@@ -0,0 +1,378 @@
+// +build linux
+
+package cgroups
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/docker/go-units"
+)
+
+const cgroupNamePrefix = "name="
+
+// https://www.kernel.org/doc/Documentation/cgroups/cgroups.txt
+func FindCgroupMountpoint(subsystem string) (string, error) {
+	// We are not using mount.GetMounts() because it's super-inefficient,
+	// parsing it directly sped up x10 times because of not using Sscanf.
+	// It was one of two major performance drawbacks in container start.
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		txt := scanner.Text()
+		fields := strings.Split(txt, " ")
+		for _, opt := range strings.Split(fields[len(fields)-1], ",") {
+			if opt == subsystem {
+				return fields[4], nil
+			}
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+
+	return "", NewNotFoundError(subsystem)
+}
+
+func FindCgroupMountpointAndRoot(subsystem string) (string, string, error) {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return "", "", err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		txt := scanner.Text()
+		fields := strings.Split(txt, " ")
+		for _, opt := range strings.Split(fields[len(fields)-1], ",") {
+			if opt == subsystem {
+				return fields[4], fields[3], nil
+			}
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return "", "", err
+	}
+
+	return "", "", NewNotFoundError(subsystem)
+}
+
+func FindCgroupMountpointDir() (string, error) {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		text := scanner.Text()
+		fields := strings.Split(text, " ")
+		// Safe as mountinfo encodes mountpoints with spaces as \040.
+		index := strings.Index(text, " - ")
+		postSeparatorFields := strings.Fields(text[index+3:])
+		numPostFields := len(postSeparatorFields)
+
+		// This is an error as we can't detect if the mount is for "cgroup"
+		if numPostFields == 0 {
+			return "", fmt.Errorf("Found no fields post '-' in %q", text)
+		}
+
+		if postSeparatorFields[0] == "cgroup" {
+			// Check that the mount is properly formated.
+			if numPostFields < 3 {
+				return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text)
+			}
+
+			return filepath.Dir(fields[4]), nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+
+	return "", NewNotFoundError("cgroup")
+}
+
+type Mount struct {
+	Mountpoint string
+	Root       string
+	Subsystems []string
+}
+
+func (m Mount) GetThisCgroupDir(cgroups map[string]string) (string, error) {
+	if len(m.Subsystems) == 0 {
+		return "", fmt.Errorf("no subsystem for mount")
+	}
+
+	return getControllerPath(m.Subsystems[0], cgroups)
+}
+
+func getCgroupMountsHelper(ss map[string]bool, mi io.Reader) ([]Mount, error) {
+	res := make([]Mount, 0, len(ss))
+	scanner := bufio.NewScanner(mi)
+	for scanner.Scan() {
+		txt := scanner.Text()
+		sepIdx := strings.Index(txt, " - ")
+		if sepIdx == -1 {
+			return nil, fmt.Errorf("invalid mountinfo format")
+		}
+		if txt[sepIdx+3:sepIdx+9] != "cgroup" {
+			continue
+		}
+		fields := strings.Split(txt, " ")
+		m := Mount{
+			Mountpoint: fields[4],
+			Root:       fields[3],
+		}
+		for _, opt := range strings.Split(fields[len(fields)-1], ",") {
+			if strings.HasPrefix(opt, cgroupNamePrefix) {
+				m.Subsystems = append(m.Subsystems, opt[len(cgroupNamePrefix):])
+			}
+			if ss[opt] {
+				m.Subsystems = append(m.Subsystems, opt)
+			}
+		}
+		res = append(res, m)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func GetCgroupMounts() ([]Mount, error) {
+	f, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	all, err := GetAllSubsystems()
+	if err != nil {
+		return nil, err
+	}
+
+	allMap := make(map[string]bool)
+	for _, s := range all {
+		allMap[s] = true
+	}
+	return getCgroupMountsHelper(allMap, f)
+}
+
+// Returns all the cgroup subsystems supported by the kernel
+func GetAllSubsystems() ([]string, error) {
+	f, err := os.Open("/proc/cgroups")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	subsystems := []string{}
+
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		if err := s.Err(); err != nil {
+			return nil, err
+		}
+		text := s.Text()
+		if text[0] != '#' {
+			parts := strings.Fields(text)
+			if len(parts) >= 4 && parts[3] != "0" {
+				subsystems = append(subsystems, parts[0])
+			}
+		}
+	}
+	return subsystems, nil
+}
+
+// Returns the relative path to the cgroup docker is running in.
+func GetThisCgroupDir(subsystem string) (string, error) {
+	cgroups, err := ParseCgroupFile("/proc/self/cgroup")
+	if err != nil {
+		return "", err
+	}
+
+	return getControllerPath(subsystem, cgroups)
+}
+
+func GetInitCgroupDir(subsystem string) (string, error) {
+
+	cgroups, err := ParseCgroupFile("/proc/1/cgroup")
+	if err != nil {
+		return "", err
+	}
+
+	return getControllerPath(subsystem, cgroups)
+}
+
+func readProcsFile(dir string) ([]int, error) {
+	f, err := os.Open(filepath.Join(dir, "cgroup.procs"))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var (
+		s   = bufio.NewScanner(f)
+		out = []int{}
+	)
+
+	for s.Scan() {
+		if t := s.Text(); t != "" {
+			pid, err := strconv.Atoi(t)
+			if err != nil {
+				return nil, err
+			}
+			out = append(out, pid)
+		}
+	}
+	return out, nil
+}
+
+func ParseCgroupFile(path string) (map[string]string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	s := bufio.NewScanner(f)
+	cgroups := make(map[string]string)
+
+	for s.Scan() {
+		if err := s.Err(); err != nil {
+			return nil, err
+		}
+
+		text := s.Text()
+		parts := strings.Split(text, ":")
+
+		for _, subs := range strings.Split(parts[1], ",") {
+			cgroups[subs] = parts[2]
+		}
+	}
+	return cgroups, nil
+}
+
+func getControllerPath(subsystem string, cgroups map[string]string) (string, error) {
+
+	if p, ok := cgroups[subsystem]; ok {
+		return p, nil
+	}
+
+	if p, ok := cgroups[cgroupNamePrefix+subsystem]; ok {
+		return p, nil
+	}
+
+	return "", NewNotFoundError(subsystem)
+}
+
+func PathExists(path string) bool {
+	if _, err := os.Stat(path); err != nil {
+		return false
+	}
+	return true
+}
+
+func EnterPid(cgroupPaths map[string]string, pid int) error {
+	for _, path := range cgroupPaths {
+		if PathExists(path) {
+			if err := ioutil.WriteFile(filepath.Join(path, "cgroup.procs"),
+				[]byte(strconv.Itoa(pid)), 0700); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// RemovePaths iterates over the provided paths removing them.
+// We trying to remove all paths five times with increasing delay between tries.
+// If after all there are not removed cgroups - appropriate error will be
+// returned.
+func RemovePaths(paths map[string]string) (err error) {
+	delay := 10 * time.Millisecond
+	for i := 0; i < 5; i++ {
+		if i != 0 {
+			time.Sleep(delay)
+			delay *= 2
+		}
+		for s, p := range paths {
+			os.RemoveAll(p)
+			// TODO: here probably should be logging
+			_, err := os.Stat(p)
+			// We need this strange way of checking cgroups existence because
+			// RemoveAll almost always returns error, even on already removed
+			// cgroups
+			if os.IsNotExist(err) {
+				delete(paths, s)
+			}
+		}
+		if len(paths) == 0 {
+			return nil
+		}
+	}
+	return fmt.Errorf("Failed to remove paths: %s", paths)
+}
+
+func GetHugePageSize() ([]string, error) {
+	var pageSizes []string
+	sizeList := []string{"B", "kB", "MB", "GB", "TB", "PB"}
+	files, err := ioutil.ReadDir("/sys/kernel/mm/hugepages")
+	if err != nil {
+		return pageSizes, err
+	}
+	for _, st := range files {
+		nameArray := strings.Split(st.Name(), "-")
+		pageSize, err := units.RAMInBytes(nameArray[1])
+		if err != nil {
+			return []string{}, err
+		}
+		sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, sizeList)
+		pageSizes = append(pageSizes, sizeString)
+	}
+
+	return pageSizes, nil
+}
+
+// GetPids returns all pids, that were added to cgroup at path.
+func GetPids(path string) ([]int, error) {
+	return readProcsFile(path)
+}
+
+// GetAllPids returns all pids, that were added to cgroup at path and to all its
+// subcgroups.
+func GetAllPids(path string) ([]int, error) {
+	var pids []int
+	// collect pids from all sub-cgroups
+	err := filepath.Walk(path, func(p string, info os.FileInfo, iErr error) error {
+		dir, file := filepath.Split(p)
+		if file != "cgroup.procs" {
+			return nil
+		}
+		if iErr != nil {
+			return iErr
+		}
+		cPids, err := readProcsFile(dir)
+		if err != nil {
+			return err
+		}
+		pids = append(pids, cPids...)
+		return nil
+	})
+	return pids, err
+}

vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils_test.go

@@ -0,0 +1,192 @@
+// +build linux
+
+package cgroups
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+const fedoraMountinfo = `15 35 0:3 / /proc rw,nosuid,nodev,noexec,relatime shared:5 - proc proc rw
+16 35 0:14 / /sys rw,nosuid,nodev,noexec,relatime shared:6 - sysfs sysfs rw,seclabel
+17 35 0:5 / /dev rw,nosuid shared:2 - devtmpfs devtmpfs rw,seclabel,size=8056484k,nr_inodes=2014121,mode=755
+18 16 0:15 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:7 - securityfs securityfs rw
+19 16 0:13 / /sys/fs/selinux rw,relatime shared:8 - selinuxfs selinuxfs rw
+20 17 0:16 / /dev/shm rw,nosuid,nodev shared:3 - tmpfs tmpfs rw,seclabel
+21 17 0:10 / /dev/pts rw,nosuid,noexec,relatime shared:4 - devpts devpts rw,seclabel,gid=5,mode=620,ptmxmode=000
+22 35 0:17 / /run rw,nosuid,nodev shared:21 - tmpfs tmpfs rw,seclabel,mode=755
+23 16 0:18 / /sys/fs/cgroup rw,nosuid,nodev,noexec shared:9 - tmpfs tmpfs rw,seclabel,mode=755
+24 23 0:19 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd
+25 16 0:20 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:20 - pstore pstore rw
+26 23 0:21 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime shared:11 - cgroup cgroup rw,cpuset,clone_children
+27 23 0:22 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:12 - cgroup cgroup rw,cpuacct,cpu,clone_children
+28 23 0:23 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:13 - cgroup cgroup rw,memory,clone_children
+29 23 0:24 / /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime shared:14 - cgroup cgroup rw,devices,clone_children
+30 23 0:25 / /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime shared:15 - cgroup cgroup rw,freezer,clone_children
+31 23 0:26 / /sys/fs/cgroup/net_cls rw,nosuid,nodev,noexec,relatime shared:16 - cgroup cgroup rw,net_cls,clone_children
+32 23 0:27 / /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime shared:17 - cgroup cgroup rw,blkio,clone_children
+33 23 0:28 / /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime shared:18 - cgroup cgroup rw,perf_event,clone_children
+34 23 0:29 / /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime shared:19 - cgroup cgroup rw,hugetlb,clone_children
+35 1 253:2 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root--f20 rw,seclabel,data=ordered
+36 15 0:30 / /proc/sys/fs/binfmt_misc rw,relatime shared:22 - autofs systemd-1 rw,fd=38,pgrp=1,timeout=300,minproto=5,maxproto=5,direct
+37 17 0:12 / /dev/mqueue rw,relatime shared:23 - mqueue mqueue rw,seclabel
+38 35 0:31 / /tmp rw shared:24 - tmpfs tmpfs rw,seclabel
+39 17 0:32 / /dev/hugepages rw,relatime shared:25 - hugetlbfs hugetlbfs rw,seclabel
+40 16 0:7 / /sys/kernel/debug rw,relatime shared:26 - debugfs debugfs rw
+41 16 0:33 / /sys/kernel/config rw,relatime shared:27 - configfs configfs rw
+42 35 0:34 / /var/lib/nfs/rpc_pipefs rw,relatime shared:28 - rpc_pipefs sunrpc rw
+43 15 0:35 / /proc/fs/nfsd rw,relatime shared:29 - nfsd sunrpc rw
+45 35 8:17 / /boot rw,relatime shared:30 - ext4 /dev/sdb1 rw,seclabel,data=ordered
+46 35 253:4 / /home rw,relatime shared:31 - ext4 /dev/mapper/ssd-home rw,seclabel,data=ordered
+47 35 253:5 / /var/lib/libvirt/images rw,noatime,nodiratime shared:32 - ext4 /dev/mapper/ssd-virt rw,seclabel,discard,data=ordered
+48 35 253:12 / /mnt/old rw,relatime shared:33 - ext4 /dev/mapper/HelpDeskRHEL6-FedoraRoot rw,seclabel,data=ordered
+121 22 0:36 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:104 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000
+124 16 0:37 / /sys/fs/fuse/connections rw,relatime shared:107 - fusectl fusectl rw
+165 38 253:3 / /tmp/mnt rw,relatime shared:147 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered
+167 35 253:15 / /var/lib/docker/devicemapper/mnt/aae4076022f0e2b80a2afbf8fc6df450c52080191fcef7fb679a73e6f073e5c2 rw,relatime shared:149 - ext4 /dev/mapper/docker-253:2-425882-aae4076022f0e2b80a2afbf8fc6df450c52080191fcef7fb679a73e6f073e5c2 rw,seclabel,discard,stripe=16,data=ordered
+171 35 253:16 / /var/lib/docker/devicemapper/mnt/c71be651f114db95180e472f7871b74fa597ee70a58ccc35cb87139ddea15373 rw,relatime shared:153 - ext4 /dev/mapper/docker-253:2-425882-c71be651f114db95180e472f7871b74fa597ee70a58ccc35cb87139ddea15373 rw,seclabel,discard,stripe=16,data=ordered
+175 35 253:17 / /var/lib/docker/devicemapper/mnt/1bac6ab72862d2d5626560df6197cf12036b82e258c53d981fa29adce6f06c3c rw,relatime shared:157 - ext4 /dev/mapper/docker-253:2-425882-1bac6ab72862d2d5626560df6197cf12036b82e258c53d981fa29adce6f06c3c rw,seclabel,discard,stripe=16,data=ordered
+179 35 253:18 / /var/lib/docker/devicemapper/mnt/d710a357d77158e80d5b2c55710ae07c94e76d34d21ee7bae65ce5418f739b09 rw,relatime shared:161 - ext4 /dev/mapper/docker-253:2-425882-d710a357d77158e80d5b2c55710ae07c94e76d34d21ee7bae65ce5418f739b09 rw,seclabel,discard,stripe=16,data=ordered
+183 35 253:19 / /var/lib/docker/devicemapper/mnt/6479f52366114d5f518db6837254baab48fab39f2ac38d5099250e9a6ceae6c7 rw,relatime shared:165 - ext4 /dev/mapper/docker-253:2-425882-6479f52366114d5f518db6837254baab48fab39f2ac38d5099250e9a6ceae6c7 rw,seclabel,discard,stripe=16,data=ordered
+187 35 253:20 / /var/lib/docker/devicemapper/mnt/8d9df91c4cca5aef49eeb2725292aab324646f723a7feab56be34c2ad08268e1 rw,relatime shared:169 - ext4 /dev/mapper/docker-253:2-425882-8d9df91c4cca5aef49eeb2725292aab324646f723a7feab56be34c2ad08268e1 rw,seclabel,discard,stripe=16,data=ordered
+191 35 253:21 / /var/lib/docker/devicemapper/mnt/c8240b768603d32e920d365dc9d1dc2a6af46cd23e7ae819947f969e1b4ec661 rw,relatime shared:173 - ext4 /dev/mapper/docker-253:2-425882-c8240b768603d32e920d365dc9d1dc2a6af46cd23e7ae819947f969e1b4ec661 rw,seclabel,discard,stripe=16,data=ordered
+195 35 253:22 / /var/lib/docker/devicemapper/mnt/2eb3a01278380bbf3ed12d86ac629eaa70a4351301ee307a5cabe7b5f3b1615f rw,relatime shared:177 - ext4 /dev/mapper/docker-253:2-425882-2eb3a01278380bbf3ed12d86ac629eaa70a4351301ee307a5cabe7b5f3b1615f rw,seclabel,discard,stripe=16,data=ordered
+199 35 253:23 / /var/lib/docker/devicemapper/mnt/37a17fb7c9d9b80821235d5f2662879bd3483915f245f9b49cdaa0e38779b70b rw,relatime shared:181 - ext4 /dev/mapper/docker-253:2-425882-37a17fb7c9d9b80821235d5f2662879bd3483915f245f9b49cdaa0e38779b70b rw,seclabel,discard,stripe=16,data=ordered
+203 35 253:24 / /var/lib/docker/devicemapper/mnt/aea459ae930bf1de913e2f29428fd80ee678a1e962d4080019d9f9774331ee2b rw,relatime shared:185 - ext4 /dev/mapper/docker-253:2-425882-aea459ae930bf1de913e2f29428fd80ee678a1e962d4080019d9f9774331ee2b rw,seclabel,discard,stripe=16,data=ordered
+207 35 253:25 / /var/lib/docker/devicemapper/mnt/928ead0bc06c454bd9f269e8585aeae0a6bd697f46dc8754c2a91309bc810882 rw,relatime shared:189 - ext4 /dev/mapper/docker-253:2-425882-928ead0bc06c454bd9f269e8585aeae0a6bd697f46dc8754c2a91309bc810882 rw,seclabel,discard,stripe=16,data=ordered
+211 35 253:26 / /var/lib/docker/devicemapper/mnt/0f284d18481d671644706e7a7244cbcf63d590d634cc882cb8721821929d0420 rw,relatime shared:193 - ext4 /dev/mapper/docker-253:2-425882-0f284d18481d671644706e7a7244cbcf63d590d634cc882cb8721821929d0420 rw,seclabel,discard,stripe=16,data=ordered
+215 35 253:27 / /var/lib/docker/devicemapper/mnt/d9dd16722ab34c38db2733e23f69e8f4803ce59658250dd63e98adff95d04919 rw,relatime shared:197 - ext4 /dev/mapper/docker-253:2-425882-d9dd16722ab34c38db2733e23f69e8f4803ce59658250dd63e98adff95d04919 rw,seclabel,discard,stripe=16,data=ordered
+219 35 253:28 / /var/lib/docker/devicemapper/mnt/bc4500479f18c2c08c21ad5282e5f826a016a386177d9874c2764751c031d634 rw,relatime shared:201 - ext4 /dev/mapper/docker-253:2-425882-bc4500479f18c2c08c21ad5282e5f826a016a386177d9874c2764751c031d634 rw,seclabel,discard,stripe=16,data=ordered
+223 35 253:29 / /var/lib/docker/devicemapper/mnt/7770c8b24eb3d5cc159a065910076938910d307ab2f5d94e1dc3b24c06ee2c8a rw,relatime shared:205 - ext4 /dev/mapper/docker-253:2-425882-7770c8b24eb3d5cc159a065910076938910d307ab2f5d94e1dc3b24c06ee2c8a rw,seclabel,discard,stripe=16,data=ordered
+227 35 253:30 / /var/lib/docker/devicemapper/mnt/c280cd3d0bf0aa36b478b292279671624cceafc1a67eaa920fa1082601297adf rw,relatime shared:209 - ext4 /dev/mapper/docker-253:2-425882-c280cd3d0bf0aa36b478b292279671624cceafc1a67eaa920fa1082601297adf rw,seclabel,discard,stripe=16,data=ordered
+231 35 253:31 / /var/lib/docker/devicemapper/mnt/8b59a7d9340279f09fea67fd6ad89ddef711e9e7050eb647984f8b5ef006335f rw,relatime shared:213 - ext4 /dev/mapper/docker-253:2-425882-8b59a7d9340279f09fea67fd6ad89ddef711e9e7050eb647984f8b5ef006335f rw,seclabel,discard,stripe=16,data=ordered
+235 35 253:32 / /var/lib/docker/devicemapper/mnt/1a28059f29eda821578b1bb27a60cc71f76f846a551abefabce6efd0146dce9f rw,relatime shared:217 - ext4 /dev/mapper/docker-253:2-425882-1a28059f29eda821578b1bb27a60cc71f76f846a551abefabce6efd0146dce9f rw,seclabel,discard,stripe=16,data=ordered
+239 35 253:33 / /var/lib/docker/devicemapper/mnt/e9aa60c60128cad1 rw,relatime shared:221 - ext4 /dev/mapper/docker-253:2-425882-e9aa60c60128cad1 rw,seclabel,discard,stripe=16,data=ordered
+243 35 253:34 / /var/lib/docker/devicemapper/mnt/5fec11304b6f4713fea7b6ccdcc1adc0a1966187f590fe25a8227428a8df275d-init rw,relatime shared:225 - ext4 /dev/mapper/docker-253:2-425882-5fec11304b6f4713fea7b6ccdcc1adc0a1966187f590fe25a8227428a8df275d-init rw,seclabel,discard,stripe=16,data=ordered
+247 35 253:35 / /var/lib/docker/devicemapper/mnt/5fec11304b6f4713fea7b6ccdcc1adc0a1966187f590fe25a8227428a8df275d rw,relatime shared:229 - ext4 /dev/mapper/docker-253:2-425882-5fec11304b6f4713fea7b6ccdcc1adc0a1966187f590fe25a8227428a8df275d rw,seclabel,discard,stripe=16,data=ordered
+31 21 0:23 / /DATA/foo_bla_bla rw,relatime - cifs //foo/BLA\040BLA\040BLA/ rw,sec=ntlm,cache=loose,unc=\\foo\BLA BLA BLA,username=my_login,domain=mydomain.com,uid=12345678,forceuid,gid=12345678,forcegid,addr=10.1.30.10,file_mode=0755,dir_mode=0755,nounix,rsize=61440,wsize=65536,actimeo=1`
+
+const systemdMountinfo = `115 83 0:32 / / rw,relatime - aufs none rw,si=c0bd3d3,dio,dirperm1
+116 115 0:35 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
+117 115 0:36 / /dev rw,nosuid - tmpfs tmpfs rw,mode=755
+118 117 0:37 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
+119 115 0:38 / /sys rw,nosuid,nodev,noexec,relatime - sysfs sysfs rw
+120 119 0:39 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755
+121 120 0:19 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd
+122 120 0:20 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,devices
+123 120 0:21 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,freezer
+124 120 0:22 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,memory
+125 120 0:23 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,net_cls,net_prio
+126 120 0:24 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,blkio
+127 120 0:25 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpuset,clone_children
+128 120 0:26 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpu,cpuacct
+129 120 0:27 /system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,perf_event,release_agent=/run/cgmanager/agents/cgm-release-agent.perf_event
+130 115 43:0 /var/lib/docker/volumes/a44a712176377f57c094397330ee04387284c478364eb25f4c3d25f775f25c26/_data /var/lib/docker rw,relatime - ext4 /dev/nbd0 rw,data=ordered
+131 115 43:0 /var/lib/docker/containers/dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/nbd0 rw,data=ordered
+132 115 43:0 /var/lib/docker/containers/dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e/hostname /etc/hostname rw,relatime - ext4 /dev/nbd0 rw,data=ordered
+133 115 43:0 /var/lib/docker/containers/dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e/hosts /etc/hosts rw,relatime - ext4 /dev/nbd0 rw,data=ordered
+134 117 0:33 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k
+135 117 0:13 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw
+136 117 0:12 /1 /dev/console rw,nosuid,noexec,relatime - devpts none rw,gid=5,mode=620,ptmxmode=000
+84 115 0:40 / /tmp rw,relatime - tmpfs none rw`
+
+func TestGetCgroupMounts(t *testing.T) {
+	type testData struct {
+		mountInfo  string
+		root       string
+		subsystems map[string]bool
+	}
+	testTable := []testData{
+		{
+			mountInfo: fedoraMountinfo,
+			root:      "/",
+			subsystems: map[string]bool{
+				"cpuset":     true,
+				"cpu":        true,
+				"cpuacct":    true,
+				"memory":     true,
+				"devices":    true,
+				"freezer":    true,
+				"net_cls":    true,
+				"blkio":      true,
+				"perf_event": true,
+				"hugetlb":    true,
+			},
+		},
+		{
+			mountInfo: systemdMountinfo,
+			root:      "/system.slice/docker-dc4eaa1a34ec4d593bc0125d31eea823a1d76ae483aeb1409cca80304e34da2e.scope",
+			subsystems: map[string]bool{
+				"cpuset":     true,
+				"cpu":        true,
+				"cpuacct":    true,
+				"memory":     true,
+				"devices":    true,
+				"freezer":    true,
+				"net_cls":    true,
+				"blkio":      true,
+				"perf_event": true,
+			},
+		},
+	}
+	for _, td := range testTable {
+		mi := bytes.NewBufferString(td.mountInfo)
+		cgMounts, err := getCgroupMountsHelper(td.subsystems, mi)
+		if err != nil {
+			t.Fatal(err)
+		}
+		cgMap := make(map[string]Mount)
+		for _, m := range cgMounts {
+			for _, ss := range m.Subsystems {
+				cgMap[ss] = m
+			}
+		}
+		for ss := range td.subsystems {
+			m, ok := cgMap[ss]
+			if !ok {
+				t.Fatalf("%s not found", ss)
+			}
+			if m.Root != td.root {
+				t.Fatalf("unexpected root for %s: %s", ss, m.Root)
+			}
+			if !strings.HasPrefix(m.Mountpoint, "/sys/fs/cgroup/") && !strings.Contains(m.Mountpoint, ss) {
+				t.Fatalf("unexpected mountpoint for %s: %s", ss, m.Mountpoint)
+			}
+			var ssFound bool
+			for _, mss := range m.Subsystems {
+				if mss == ss {
+					ssFound = true
+					break
+				}
+			}
+			if !ssFound {
+				t.Fatalf("subsystem %s not found in Subsystems field %v", ss, m.Subsystems)
+			}
+		}
+	}
+}
+
+func BenchmarkGetCgroupMounts(b *testing.B) {
+	subsystems := map[string]bool{
+		"cpuset":     true,
+		"cpu":        true,
+		"cpuacct":    true,
+		"memory":     true,
+		"devices":    true,
+		"freezer":    true,
+		"net_cls":    true,
+		"blkio":      true,
+		"perf_event": true,
+		"hugetlb":    true,
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		mi := bytes.NewBufferString(fedoraMountinfo)
+		b.StartTimer()
+		if _, err := getCgroupMountsHelper(subsystems, mi); err != nil {
+			b.Fatal(err)
+		}
+	}
+}