From 29bd1c79ddd452ba2ebc2aae4f0e20760700b2e8 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Tue, 26 Sep 2017 19:58:51 +0000 Subject: [PATCH 1/4] Vendor in container storage This should add quota support to cri-o Signed-off-by: Daniel J Walsh --- docs/crio.conf.5.md | 4 +- vendor.conf | 2 +- .../containers/storage/containers.go | 2 + .../containers/storage/drivers/aufs/aufs.go | 243 +++++--- .../containers/storage/drivers/aufs/dirs.go | 2 +- .../containers/storage/drivers/aufs/mount.go | 4 +- .../storage/drivers/aufs/mount_linux.go | 4 +- .../storage/drivers/aufs/mount_unsupported.go | 2 +- .../containers/storage/drivers/btrfs/btrfs.go | 264 ++++++-- .../containers/storage/drivers/counter.go | 34 +- .../storage/drivers/devmapper/device_setup.go | 236 +++++++ .../storage/drivers/devmapper/deviceset.go | 574 +++++++++++------- .../storage/drivers/devmapper/driver.go | 54 +- .../storage/drivers/devmapper/mount.go | 13 +- .../containers/storage/drivers/driver.go | 86 ++- .../storage/drivers/driver_freebsd.go | 8 +- .../storage/drivers/driver_linux.go | 15 +- .../storage/drivers/driver_solaris.go | 44 +- .../containers/storage/drivers/fsdiff.go | 52 +- .../storage/drivers/overlay/check.go | 79 +++ .../storage/drivers/overlay/mount.go | 12 +- .../storage/drivers/overlay/overlay.go | 336 +++++++--- .../storage/drivers/overlay/randomid.go | 3 +- .../drivers/overlayutils/overlayutils.go | 18 + .../containers/storage/drivers/plugin.go | 32 - .../storage/drivers/plugin_unsupported.go | 7 - .../containers/storage/drivers/proxy.go | 226 ------- .../storage/drivers/quota/projectquota.go | 337 ++++++++++ .../containers/storage/drivers/vfs/driver.go | 53 +- .../storage/drivers/windows/windows.go | 550 +++++++++++------ .../containers/storage/drivers/zfs/zfs.go | 42 +- .../storage/drivers/zfs/zfs_freebsd.go | 6 +- .../storage/drivers/zfs/zfs_linux.go | 6 +- .../storage/drivers/zfs/zfs_solaris.go | 9 +- .../github.com/containers/storage/images.go | 7 +- .../github.com/containers/storage/layers.go | 10 +- .../storage/pkg/archive/example_changes.go | 97 +++ .../storage/pkg/dmesg/dmesg_linux.go | 20 + .../storage/pkg/fsutils/fsutils_linux.go | 88 +++ .../containers/storage/pkg/locker/README.md | 65 ++ .../containers/storage/pkg/locker/locker.go | 112 ++++ .../containers/storage/pkg/plugins/client.go | 205 ------- .../storage/pkg/plugins/discovery.go | 131 ---- .../storage/pkg/plugins/discovery_unix.go | 5 - .../storage/pkg/plugins/discovery_windows.go | 8 - .../containers/storage/pkg/plugins/errors.go | 33 - .../containers/storage/pkg/plugins/plugins.go | 329 ---------- .../storage/pkg/plugins/plugins_unix.go | 9 - .../storage/pkg/plugins/plugins_windows.go | 8 - .../storage/pkg/plugins/transport/http.go | 36 -- .../pkg/plugins/transport/transport.go | 36 -- vendor/github.com/containers/storage/store.go | 74 ++- 52 files changed, 2751 insertions(+), 1881 deletions(-) create mode 100644 vendor/github.com/containers/storage/drivers/devmapper/device_setup.go create mode 100644 vendor/github.com/containers/storage/drivers/overlay/check.go create mode 100644 vendor/github.com/containers/storage/drivers/overlayutils/overlayutils.go delete mode 100644 vendor/github.com/containers/storage/drivers/plugin.go delete mode 100644 vendor/github.com/containers/storage/drivers/plugin_unsupported.go delete mode 100644 vendor/github.com/containers/storage/drivers/proxy.go create mode 100644 vendor/github.com/containers/storage/drivers/quota/projectquota.go create mode 100644 vendor/github.com/containers/storage/pkg/archive/example_changes.go create mode 100644 vendor/github.com/containers/storage/pkg/dmesg/dmesg_linux.go create mode 100644 vendor/github.com/containers/storage/pkg/fsutils/fsutils_linux.go create mode 100644 vendor/github.com/containers/storage/pkg/locker/README.md create mode 100644 vendor/github.com/containers/storage/pkg/locker/locker.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/client.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/discovery.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/discovery_unix.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/discovery_windows.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/errors.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/plugins.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/plugins_unix.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/plugins_windows.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/transport/http.go delete mode 100644 vendor/github.com/containers/storage/pkg/plugins/transport/transport.go diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index dca22196..01d1bc32 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -36,7 +36,7 @@ The `crio` table supports the following options: CRIO state dir (default: "/var/run/containers/storage") **storage_driver**="" - CRIO storage driver (default is "devicemapper") + CRIO storage driver (default is "overlay") **storage_option**=[] CRIO storage driver option list (no default) @@ -125,7 +125,7 @@ The `crio` table supports the following options: Path to CNI plugin binaries (default: "/opt/cni/bin/") # SEE ALSO -crio(8) +crio(8), containers-storage.conf(5) # HISTORY Oct 2016, Originally compiled by Aleksa Sarai diff --git a/vendor.conf b/vendor.conf index 0aae101f..631391bf 100644 --- a/vendor.conf +++ b/vendor.conf @@ -7,7 +7,7 @@ github.com/sirupsen/logrus v1.0.0 github.com/containers/image d17474f39dae1da15ab9ae033d57ebefcf62f77a github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 github.com/ostreedev/ostree-go master -github.com/containers/storage 9c85fa701316a49afdf85d55a0d7cb582ed03625 +github.com/containers/storage 64bf27465d0d1edd89e7a4ce49866fea01145782 github.com/containernetworking/cni v0.4.0 google.golang.org/grpc v1.0.4 https://github.com/grpc/grpc-go github.com/opencontainers/selinux v1.0.0-rc1 diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index 31611681..c6eb9eaf 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -239,6 +239,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat if _, idInUse := r.byid[id]; idInUse { return nil, ErrDuplicateID } + names = dedupeNames(names) for _, name := range names { if _, nameInUse := r.byname[name]; nameInUse { return nil, ErrDuplicateName @@ -288,6 +289,7 @@ func (r *containerStore) removeName(container *Container, name string) { } func (r *containerStore) SetNames(id string, names []string) error { + names = dedupeNames(names) if container, ok := r.lookup(id); ok { for _, name := range container.Names { delete(r.byname, name) diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index 7301dbdc..aa0da7ad 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -33,22 +33,22 @@ import ( "path/filepath" "strings" "sync" - "syscall" - - "github.com/sirupsen/logrus" - "github.com/vbatts/tar-split/tar/storage" + "time" "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/locker" mountpk "github.com/containers/storage/pkg/mount" - "github.com/containers/storage/pkg/stringid" - + "github.com/containers/storage/pkg/system" rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vbatts/tar-split/tar/storage" + "golang.org/x/sys/unix" ) var ( @@ -75,6 +75,8 @@ type Driver struct { ctr *graphdriver.RefCounter pathCacheLock sync.Mutex pathCache map[string]string + naiveDiff graphdriver.DiffDriver + locker *locker.Locker } // Init returns a new AUFS driver. @@ -84,6 +86,7 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap // Try to load the aufs kernel module if err := supportsAufs(); err != nil { return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel does not support aufs") + } fsMagic, err := graphdriver.GetFSMagic(root) @@ -112,6 +115,7 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap gidMaps: gidMaps, pathCache: make(map[string]string), ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicAufs)), + locker: locker.New(), } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) @@ -138,6 +142,32 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } } + logger := logrus.WithFields(logrus.Fields{ + "module": "graphdriver", + "driver": "aufs", + }) + + for _, path := range []string{"mnt", "diff"} { + p := filepath.Join(root, path) + entries, err := ioutil.ReadDir(p) + if err != nil { + logger.WithError(err).WithField("dir", p).Error("error reading dir entries") + continue + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if strings.HasSuffix(entry.Name(), "-removing") { + logger.WithField("dir", entry.Name()).Debug("Cleaning up stale layer dir") + if err := system.EnsureRemoveAll(filepath.Join(p, entry.Name())); err != nil { + logger.WithField("dir", entry.Name()).WithError(err).Error("Error removing stale layer dir") + } + } + } + } + + a.naiveDiff = graphdriver.NewNaiveDiffDriver(a, uidMaps, gidMaps) return a, nil } @@ -201,17 +231,22 @@ func (a *Driver) Exists(id string) bool { return true } +// AdditionalImageStores returns additional image stores supported by the driver +func (a *Driver) AdditionalImageStores() []string { + return nil +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (a *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return a.Create(id, parent, mountLabel, storageOpt) +func (a *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return a.Create(id, parent, opts) } // Create three folders for each id // mnt, layers, and diff -func (a *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { +func (a *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { - if len(storageOpt) != 0 { + if opts != nil && len(opts.StorageOpt) != 0 { return fmt.Errorf("--storage-opt is not supported for aufs") } @@ -226,7 +261,7 @@ func (a *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str defer f.Close() if parent != "" { - ids, err := getParentIds(a.rootPath(), parent) + ids, err := getParentIDs(a.rootPath(), parent) if err != nil { return err } @@ -269,35 +304,68 @@ func (a *Driver) createDirsFor(id string) error { // Remove will unmount and remove the given id. func (a *Driver) Remove(id string) error { + a.locker.Lock(id) + defer a.locker.Unlock(id) a.pathCacheLock.Lock() mountpoint, exists := a.pathCache[id] a.pathCacheLock.Unlock() if !exists { mountpoint = a.getMountpoint(id) } - if err := a.unmount(mountpoint); err != nil { - // no need to return here, we can still try to remove since the `Rename` will fail below if still mounted - logrus.Debugf("aufs: error while unmounting %s: %v", mountpoint, err) - } - // Atomically remove each directory in turn by first moving it out of the - // way (so that container runtimes don't find it anymore) before doing removal of - // the whole tree. - tmpMntPath := path.Join(a.mntPath(), fmt.Sprintf("%s-removing", id)) - if err := os.Rename(mountpoint, tmpMntPath); err != nil && !os.IsNotExist(err) { - return err - } - defer os.RemoveAll(tmpMntPath) + logger := logrus.WithFields(logrus.Fields{ + "module": "graphdriver", + "driver": "aufs", + "layer": id, + }) - tmpDiffpath := path.Join(a.diffPath(), fmt.Sprintf("%s-removing", id)) - if err := os.Rename(a.getDiffPath(id), tmpDiffpath); err != nil && !os.IsNotExist(err) { - return err + var retries int + for { + mounted, err := a.mounted(mountpoint) + if err != nil { + if os.IsNotExist(err) { + break + } + return err + } + if !mounted { + break + } + + err = a.unmount(mountpoint) + if err == nil { + break + } + + if err != unix.EBUSY { + return errors.Wrapf(err, "aufs: unmount error: %s", mountpoint) + } + if retries >= 5 { + return errors.Wrapf(err, "aufs: unmount error after retries: %s", mountpoint) + } + // If unmount returns EBUSY, it could be a transient error. Sleep and retry. + retries++ + logger.Warnf("unmount failed due to EBUSY: retry count: %d", retries) + time.Sleep(100 * time.Millisecond) } - defer os.RemoveAll(tmpDiffpath) // Remove the layers file for the id if err := os.Remove(path.Join(a.rootPath(), "layers", id)); err != nil && !os.IsNotExist(err) { - return err + return errors.Wrapf(err, "error removing layers dir for %s", id) + } + + if err := atomicRemove(a.getDiffPath(id)); err != nil { + return errors.Wrapf(err, "could not remove diff path for id %s", id) + } + + // Atomically remove each directory in turn by first moving it out of the + // way (so that container runtime doesn't find it anymore) before doing removal of + // the whole tree. + if err := atomicRemove(mountpoint); err != nil { + if errors.Cause(err) == unix.EBUSY { + logger.WithField("dir", mountpoint).WithError(err).Warn("error performing atomic remove due to EBUSY") + } + return errors.Wrapf(err, "could not remove mountpoint for id %s", id) } a.pathCacheLock.Lock() @@ -306,9 +374,29 @@ func (a *Driver) Remove(id string) error { return nil } +func atomicRemove(source string) error { + target := source + "-removing" + + err := os.Rename(source, target) + switch { + case err == nil, os.IsNotExist(err): + case os.IsExist(err): + // Got error saying the target dir already exists, maybe the source doesn't exist due to a previous (failed) remove + if _, e := os.Stat(source); !os.IsNotExist(e) { + return errors.Wrapf(err, "target rename dir '%s' exists but should not, this needs to be manually cleaned up") + } + default: + return errors.Wrapf(err, "error preparing atomic delete") + } + + return system.EnsureRemoveAll(target) +} + // Get returns the rootfs path for the id. -// This will mount the dir at it's given path +// This will mount the dir at its given path func (a *Driver) Get(id, mountLabel string) (string, error) { + a.locker.Lock(id) + defer a.locker.Unlock(id) parents, err := a.getParentLayerPaths(id) if err != nil && !os.IsNotExist(err) { return "", err @@ -344,6 +432,8 @@ func (a *Driver) Get(id, mountLabel string) (string, error) { // Put unmounts and updates list of active mounts. func (a *Driver) Put(id string) error { + a.locker.Lock(id) + defer a.locker.Unlock(id) a.pathCacheLock.Lock() m, exists := a.pathCache[id] if !exists { @@ -362,9 +452,22 @@ func (a *Driver) Put(id string) error { return err } +// isParent returns if the passed in parent is the direct parent of the passed in layer +func (a *Driver) isParent(id, parent string) bool { + parents, _ := getParentIDs(a.rootPath(), id) + if parent == "" && len(parents) > 0 { + return false + } + return !(len(parents) > 0 && parent != parents[0]) +} + // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". func (a *Driver) Diff(id, parent string) (io.ReadCloser, error) { + if !a.isParent(id, parent) { + return a.naiveDiff.Diff(id, parent) + } + // AUFS doesn't need the parent layer to produce a diff. return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ Compression: archive.Uncompressed, @@ -374,12 +477,6 @@ func (a *Driver) Diff(id, parent string) (io.ReadCloser, error) { }) } -// AdditionalImageStores returns additional image stores supported by the driver -func (a *Driver) AdditionalImageStores() []string { - var imageStores []string - return imageStores -} - type fileGetNilCloser struct { storage.FileGetter } @@ -406,6 +503,9 @@ func (a *Driver) applyDiff(id string, diff io.Reader) error { // and its parent and returns the size in bytes of the changes // relative to its base filesystem directory. func (a *Driver) DiffSize(id, parent string) (size int64, err error) { + if !a.isParent(id, parent) { + return a.naiveDiff.DiffSize(id, parent) + } // AUFS doesn't need the parent layer to calculate the diff size. return directory.Size(path.Join(a.rootPath(), "diff", id)) } @@ -414,7 +514,11 @@ func (a *Driver) DiffSize(id, parent string) (size int64, err error) { // layer with the specified id and parent, returning the size of the // new layer in bytes. func (a *Driver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) { - // AUFS doesn't need the parent id to apply the diff. + if !a.isParent(id, parent) { + return a.naiveDiff.ApplyDiff(id, parent, diff) + } + + // AUFS doesn't need the parent id to apply the diff if it is the direct parent. if err = a.applyDiff(id, diff); err != nil { return } @@ -425,6 +529,10 @@ func (a *Driver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err e // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. func (a *Driver) Changes(id, parent string) ([]archive.Change, error) { + if !a.isParent(id, parent) { + return a.naiveDiff.Changes(id, parent) + } + // AUFS doesn't have snapshots, so we need to get changes from all parent // layers. layers, err := a.getParentLayerPaths(id) @@ -435,7 +543,7 @@ func (a *Driver) Changes(id, parent string) ([]archive.Change, error) { } func (a *Driver) getParentLayerPaths(id string) ([]string, error) { - parentIds, err := getParentIds(a.rootPath(), id) + parentIds, err := getParentIDs(a.rootPath(), id) if err != nil { return nil, err } @@ -500,7 +608,7 @@ func (a *Driver) Cleanup() error { for _, m := range dirs { if err := a.unmount(m); err != nil { - logrus.Debugf("aufs error unmounting %s: %s", stringid.TruncateID(m), err) + logrus.Debugf("aufs error unmounting %s: %s", m, err) } } return mountpk.Unmount(a.root) @@ -518,46 +626,35 @@ func (a *Driver) aufsMount(ro []string, rw, target, mountLabel string) (err erro offset := 54 if useDirperm() { - offset += len("dirperm1") + offset += len(",dirperm1") } - b := make([]byte, syscall.Getpagesize()-len(mountLabel)-offset) // room for xino & mountLabel + b := make([]byte, unix.Getpagesize()-len(mountLabel)-offset) // room for xino & mountLabel bp := copy(b, fmt.Sprintf("br:%s=rw", rw)) - firstMount := true - i := 0 - - for { - for ; i < len(ro); i++ { - layer := fmt.Sprintf(":%s=ro+wh", ro[i]) - - if firstMount { - if bp+len(layer) > len(b) { - break - } - bp += copy(b[bp:], layer) - } else { - data := label.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel) - if err = mount("none", target, "aufs", syscall.MS_REMOUNT, data); err != nil { - return - } - } - } - - if firstMount { - opts := "dio,xino=/dev/shm/aufs.xino" - if useDirperm() { - opts += ",dirperm1" - } - data := label.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel) - if err = mount("none", target, "aufs", 0, data); err != nil { - return - } - firstMount = false - } - - if i == len(ro) { + index := 0 + for ; index < len(ro); index++ { + layer := fmt.Sprintf(":%s=ro+wh", ro[index]) + if bp+len(layer) > len(b) { break } + bp += copy(b[bp:], layer) + } + + opts := "dio,xino=/dev/shm/aufs.xino" + if useDirperm() { + opts += ",dirperm1" + } + data := label.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel) + if err = mount("none", target, "aufs", 0, data); err != nil { + return + } + + for ; index < len(ro); index++ { + layer := fmt.Sprintf(":%s=ro+wh", ro[index]) + data := label.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel) + if err = mount("none", target, "aufs", unix.MS_REMOUNT, data); err != nil { + return + } } return diff --git a/vendor/github.com/containers/storage/drivers/aufs/dirs.go b/vendor/github.com/containers/storage/drivers/aufs/dirs.go index eb298d9e..d2325fc4 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/dirs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/dirs.go @@ -29,7 +29,7 @@ func loadIds(root string) ([]string, error) { // // If there are no lines in the file then the id has no parent // and an empty slice is returned. -func getParentIds(root, id string) ([]string, error) { +func getParentIDs(root, id string) ([]string, error) { f, err := os.Open(path.Join(root, "layers", id)) if err != nil { return nil, err diff --git a/vendor/github.com/containers/storage/drivers/aufs/mount.go b/vendor/github.com/containers/storage/drivers/aufs/mount.go index 8314f142..100e7537 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/mount.go +++ b/vendor/github.com/containers/storage/drivers/aufs/mount.go @@ -4,9 +4,9 @@ package aufs import ( "os/exec" - "syscall" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // Unmount the target specified. @@ -14,7 +14,7 @@ func Unmount(target string) error { if err := exec.Command("auplink", target, "flush").Run(); err != nil { logrus.Warnf("Couldn't run auplink before unmount %s: %s", target, err) } - if err := syscall.Unmount(target, 0); err != nil { + if err := unix.Unmount(target, 0); err != nil { return err } return nil diff --git a/vendor/github.com/containers/storage/drivers/aufs/mount_linux.go b/vendor/github.com/containers/storage/drivers/aufs/mount_linux.go index 8062bae4..937104ba 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/mount_linux.go +++ b/vendor/github.com/containers/storage/drivers/aufs/mount_linux.go @@ -1,7 +1,7 @@ package aufs -import "syscall" +import "golang.org/x/sys/unix" func mount(source string, target string, fstype string, flags uintptr, data string) error { - return syscall.Mount(source, target, fstype, flags, data) + return unix.Mount(source, target, fstype, flags, data) } diff --git a/vendor/github.com/containers/storage/drivers/aufs/mount_unsupported.go b/vendor/github.com/containers/storage/drivers/aufs/mount_unsupported.go index c807902d..d030b066 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/mount_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/aufs/mount_unsupported.go @@ -2,7 +2,7 @@ package aufs -import "github.com/pkg/errors" +import "errors" // MsRemount declared to specify a non-linux system mount. const MsRemount = 0 diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index 9e16f894..abc856c8 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -16,31 +16,32 @@ import "C" import ( "fmt" + "io/ioutil" + "math" "os" "path" "path/filepath" + "strconv" "strings" - "syscall" + "sync" "unsafe" "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" + "github.com/containers/storage/pkg/system" "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) func init() { graphdriver.Register("btrfs", Init) } -var ( - quotaEnabled = false - userDiskQuota = false -) - type btrfsOptions struct { minSpace uint64 size uint64 @@ -71,18 +72,11 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - opt, err := parseOptions(options) + opt, userDiskQuota, err := parseOptions(options) if err != nil { return nil, err } - if userDiskQuota { - if err := subvolEnableQuota(home); err != nil { - return nil, err - } - quotaEnabled = true - } - driver := &Driver{ home: home, uidMaps: uidMaps, @@ -90,39 +84,48 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap options: opt, } + if userDiskQuota { + if err := driver.subvolEnableQuota(); err != nil { + return nil, err + } + } + return graphdriver.NewNaiveDiffDriver(driver, uidMaps, gidMaps), nil } -func parseOptions(opt []string) (btrfsOptions, error) { +func parseOptions(opt []string) (btrfsOptions, bool, error) { var options btrfsOptions + userDiskQuota := false for _, option := range opt { key, val, err := parsers.ParseKeyValueOpt(option) if err != nil { - return options, err + return options, userDiskQuota, err } key = strings.ToLower(key) switch key { case "btrfs.min_space": minSpace, err := units.RAMInBytes(val) if err != nil { - return options, err + return options, userDiskQuota, err } userDiskQuota = true options.minSpace = uint64(minSpace) default: - return options, fmt.Errorf("Unknown option %s", key) + return options, userDiskQuota, fmt.Errorf("Unknown option %s", key) } } - return options, nil + return options, userDiskQuota, nil } // Driver contains information about the filesystem mounted. type Driver struct { //root of the file system - home string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap - options btrfsOptions + home string + uidMaps []idtools.IDMap + gidMaps []idtools.IDMap + options btrfsOptions + quotaEnabled bool + once sync.Once } // String prints the name of the driver (btrfs). @@ -151,10 +154,8 @@ func (d *Driver) Metadata(id string) (map[string]string, error) { // Cleanup unmounts the home directory. func (d *Driver) Cleanup() error { - if quotaEnabled { - if err := subvolDisableQuota(d.home); err != nil { - return err - } + if err := d.subvolDisableQuota(); err != nil { + return err } return mount.Unmount(d.home) @@ -197,7 +198,7 @@ func subvolCreate(path, name string) error { args.name[i] = C.char(c) } - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SUBVOL_CREATE, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to create btrfs subvolume: %v", errno.Error()) @@ -225,7 +226,7 @@ func subvolSnapshot(src, dest, name string) error { C.set_name_btrfs_ioctl_vol_args_v2(&args, cs) C.free(unsafe.Pointer(cs)) - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(destDir), C.BTRFS_IOC_SNAP_CREATE_V2, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to create btrfs snapshot: %v", errno.Error()) @@ -234,8 +235,8 @@ func subvolSnapshot(src, dest, name string) error { } func isSubvolume(p string) (bool, error) { - var bufStat syscall.Stat_t - if err := syscall.Lstat(p, &bufStat); err != nil { + var bufStat unix.Stat_t + if err := unix.Lstat(p, &bufStat); err != nil { return false, err } @@ -243,7 +244,7 @@ func isSubvolume(p string) (bool, error) { return bufStat.Ino == C.BTRFS_FIRST_FREE_OBJECTID, nil } -func subvolDelete(dirpath, name string) error { +func subvolDelete(dirpath, name string, quotaEnabled bool) error { dir, err := openDir(dirpath) if err != nil { return err @@ -271,7 +272,7 @@ func subvolDelete(dirpath, name string) error { return fmt.Errorf("Failed to test if %s is a btrfs subvolume: %v", p, err) } if sv { - if err := subvolDelete(path.Dir(p), f.Name()); err != nil { + if err := subvolDelete(path.Dir(p), f.Name(), quotaEnabled); err != nil { return fmt.Errorf("Failed to destroy btrfs child subvolume (%s) of parent (%s): %v", p, dirpath, err) } } @@ -282,12 +283,27 @@ func subvolDelete(dirpath, name string) error { return fmt.Errorf("Recursively walking subvolumes for %s failed: %v", dirpath, err) } + if quotaEnabled { + if qgroupid, err := subvolLookupQgroup(fullPath); err == nil { + var args C.struct_btrfs_ioctl_qgroup_create_args + args.qgroupid = C.__u64(qgroupid) + + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_CREATE, + uintptr(unsafe.Pointer(&args))) + if errno != 0 { + logrus.Errorf("Failed to delete btrfs qgroup %v for %s: %v", qgroupid, fullPath, errno.Error()) + } + } else { + logrus.Errorf("Failed to lookup btrfs qgroup for %s: %v", fullPath, err.Error()) + } + } + // all subvolumes have been removed // now remove the one originally passed in for i, c := range []byte(name) { args.name[i] = C.char(c) } - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_SNAP_DESTROY, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to destroy btrfs snapshot %s for %s: %v", dirpath, name, errno.Error()) @@ -295,8 +311,27 @@ func subvolDelete(dirpath, name string) error { return nil } -func subvolEnableQuota(path string) error { - dir, err := openDir(path) +func (d *Driver) updateQuotaStatus() { + d.once.Do(func() { + if !d.quotaEnabled { + // In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed + if err := subvolQgroupStatus(d.home); err != nil { + // quota is still not enabled + return + } + d.quotaEnabled = true + } + }) +} + +func (d *Driver) subvolEnableQuota() error { + d.updateQuotaStatus() + + if d.quotaEnabled { + return nil + } + + dir, err := openDir(d.home) if err != nil { return err } @@ -304,17 +339,25 @@ func subvolEnableQuota(path string) error { var args C.struct_btrfs_ioctl_quota_ctl_args args.cmd = C.BTRFS_QUOTA_CTL_ENABLE - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to enable btrfs quota for %s: %v", dir, errno.Error()) } + d.quotaEnabled = true + return nil } -func subvolDisableQuota(path string) error { - dir, err := openDir(path) +func (d *Driver) subvolDisableQuota() error { + d.updateQuotaStatus() + + if !d.quotaEnabled { + return nil + } + + dir, err := openDir(d.home) if err != nil { return err } @@ -322,24 +365,32 @@ func subvolDisableQuota(path string) error { var args C.struct_btrfs_ioctl_quota_ctl_args args.cmd = C.BTRFS_QUOTA_CTL_DISABLE - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to disable btrfs quota for %s: %v", dir, errno.Error()) } + d.quotaEnabled = false + return nil } -func subvolRescanQuota(path string) error { - dir, err := openDir(path) +func (d *Driver) subvolRescanQuota() error { + d.updateQuotaStatus() + + if !d.quotaEnabled { + return nil + } + + dir, err := openDir(d.home) if err != nil { return err } defer closeDir(dir) var args C.struct_btrfs_ioctl_quota_rescan_args - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_RESCAN_WAIT, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_RESCAN_WAIT, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to rescan btrfs quota for %s: %v", dir, errno.Error()) @@ -358,7 +409,7 @@ func subvolLimitQgroup(path string, size uint64) error { var args C.struct_btrfs_ioctl_qgroup_limit_args args.lim.max_referenced = C.__u64(size) args.lim.flags = C.BTRFS_QGROUP_LIMIT_MAX_RFER - _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_LIMIT, + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QGROUP_LIMIT, uintptr(unsafe.Pointer(&args))) if errno != 0 { return fmt.Errorf("Failed to limit qgroup for %s: %v", dir, errno.Error()) @@ -367,6 +418,60 @@ func subvolLimitQgroup(path string, size uint64) error { return nil } +// subvolQgroupStatus performs a BTRFS_IOC_TREE_SEARCH on the root path +// with search key of BTRFS_QGROUP_STATUS_KEY. +// In case qgroup is enabled, the retuned key type will match BTRFS_QGROUP_STATUS_KEY. +// For more details please see https://github.com/kdave/btrfs-progs/blob/v4.9/qgroup.c#L1035 +func subvolQgroupStatus(path string) error { + dir, err := openDir(path) + if err != nil { + return err + } + defer closeDir(dir) + + var args C.struct_btrfs_ioctl_search_args + args.key.tree_id = C.BTRFS_QUOTA_TREE_OBJECTID + args.key.min_type = C.BTRFS_QGROUP_STATUS_KEY + args.key.max_type = C.BTRFS_QGROUP_STATUS_KEY + args.key.max_objectid = C.__u64(math.MaxUint64) + args.key.max_offset = C.__u64(math.MaxUint64) + args.key.max_transid = C.__u64(math.MaxUint64) + args.key.nr_items = 4096 + + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_TREE_SEARCH, + uintptr(unsafe.Pointer(&args))) + if errno != 0 { + return fmt.Errorf("Failed to search qgroup for %s: %v", path, errno.Error()) + } + sh := (*C.struct_btrfs_ioctl_search_header)(unsafe.Pointer(&args.buf)) + if sh._type != C.BTRFS_QGROUP_STATUS_KEY { + return fmt.Errorf("Invalid qgroup search header type for %s: %v", path, sh._type) + } + return nil +} + +func subvolLookupQgroup(path string) (uint64, error) { + dir, err := openDir(path) + if err != nil { + return 0, err + } + defer closeDir(dir) + + var args C.struct_btrfs_ioctl_ino_lookup_args + args.objectid = C.BTRFS_FIRST_FREE_OBJECTID + + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_INO_LOOKUP, + uintptr(unsafe.Pointer(&args))) + if errno != 0 { + return 0, fmt.Errorf("Failed to lookup qgroup for %s: %v", dir, errno.Error()) + } + if args.treeid == 0 { + return 0, fmt.Errorf("Invalid qgroup id for %s: 0", dir) + } + + return uint64(args.treeid), nil +} + func (d *Driver) subvolumesDir() string { return path.Join(d.home, "subvolumes") } @@ -375,14 +480,23 @@ func (d *Driver) subvolumesDirID(id string) string { return path.Join(d.subvolumesDir(), id) } +func (d *Driver) quotasDir() string { + return path.Join(d.home, "quotas") +} + +func (d *Driver) quotasDirID(id string) string { + return path.Join(d.quotasDir(), id) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return d.Create(id, parent, opts) } // Create the filesystem with given id. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + quotas := path.Join(d.home, "quotas") subvolumes := path.Join(d.home, "subvolumes") rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) if err != nil { @@ -409,14 +523,26 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str } } + var storageOpt map[string]string + if opts != nil { + storageOpt = opts.StorageOpt + } + if _, ok := storageOpt["size"]; ok { driver := &Driver{} if err := d.parseStorageOpt(storageOpt, driver); err != nil { return err } + if err := d.setStorageSize(path.Join(subvolumes, id), driver); err != nil { return err } + if err := idtools.MkdirAllAs(quotas, 0700, rootUID, rootGID); err != nil { + return err + } + if err := ioutil.WriteFile(path.Join(quotas, id), []byte(fmt.Sprint(driver.options.size)), 0644); err != nil { + return err + } } // if we have a remapped root (user namespaces enabled), change the created snapshot @@ -427,6 +553,11 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str } } + mountLabel := "" + if opts != nil { + mountLabel = opts.MountLabel + } + return label.Relabel(path.Join(subvolumes, id), mountLabel, false) } @@ -459,11 +590,8 @@ func (d *Driver) setStorageSize(dir string, driver *Driver) error { return fmt.Errorf("btrfs: storage size cannot be less than %s", units.HumanSize(float64(d.options.minSpace))) } - if !quotaEnabled { - if err := subvolEnableQuota(d.home); err != nil { - return err - } - quotaEnabled = true + if err := d.subvolEnableQuota(); err != nil { + return err } if err := subvolLimitQgroup(dir, driver.options.size); err != nil { @@ -479,13 +607,25 @@ func (d *Driver) Remove(id string) error { if _, err := os.Stat(dir); err != nil { return err } - if err := subvolDelete(d.subvolumesDir(), id); err != nil { + quotasDir := d.quotasDirID(id) + if _, err := os.Stat(quotasDir); err == nil { + if err := os.Remove(quotasDir); err != nil { + return err + } + } else if !os.IsNotExist(err) { return err } - if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { + + // Call updateQuotaStatus() to invoke status update + d.updateQuotaStatus() + + if err := subvolDelete(d.subvolumesDir(), id, d.quotaEnabled); err != nil { return err } - if err := subvolRescanQuota(d.home); err != nil { + if err := system.EnsureRemoveAll(dir); err != nil { + return err + } + if err := d.subvolRescanQuota(); err != nil { return err } return nil @@ -503,6 +643,17 @@ func (d *Driver) Get(id, mountLabel string) (string, error) { return "", fmt.Errorf("%s: not a directory", dir) } + if quota, err := ioutil.ReadFile(d.quotasDirID(id)); err == nil { + if size, err := strconv.ParseUint(string(quota), 10, 64); err == nil && size >= d.options.minSpace { + if err := d.subvolEnableQuota(); err != nil { + return "", err + } + if err := subvolLimitQgroup(dir, size); err != nil { + return "", err + } + } + } + return dir, nil } @@ -522,6 +673,5 @@ func (d *Driver) Exists(id string) bool { // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { - var imageStores []string - return imageStores + return nil } diff --git a/vendor/github.com/containers/storage/drivers/counter.go b/vendor/github.com/containers/storage/drivers/counter.go index 5ea604f5..72551a38 100644 --- a/vendor/github.com/containers/storage/drivers/counter.go +++ b/vendor/github.com/containers/storage/drivers/counter.go @@ -22,30 +22,21 @@ func NewRefCounter(c Checker) *RefCounter { } } -// Increment increaes the ref count for the given id and returns the current count +// Increment increases the ref count for the given id and returns the current count func (c *RefCounter) Increment(path string) int { - c.mu.Lock() - m := c.counts[path] - if m == nil { - m = &minfo{} - c.counts[path] = m - } - // if we are checking this path for the first time check to make sure - // if it was already mounted on the system and make sure we have a correct ref - // count if it is mounted as it is in use. - if !m.check { - m.check = true - if c.checker.IsMounted(path) { - m.count++ - } - } - m.count++ - c.mu.Unlock() - return m.count + return c.incdec(path, func(minfo *minfo) { + minfo.count++ + }) } // Decrement decreases the ref count for the given id and returns the current count func (c *RefCounter) Decrement(path string) int { + return c.incdec(path, func(minfo *minfo) { + minfo.count-- + }) +} + +func (c *RefCounter) incdec(path string, infoOp func(minfo *minfo)) int { c.mu.Lock() m := c.counts[path] if m == nil { @@ -61,7 +52,8 @@ func (c *RefCounter) Decrement(path string) int { m.count++ } } - m.count-- + infoOp(m) + count := m.count c.mu.Unlock() - return m.count + return count } diff --git a/vendor/github.com/containers/storage/drivers/devmapper/device_setup.go b/vendor/github.com/containers/storage/drivers/devmapper/device_setup.go new file mode 100644 index 00000000..1430c885 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/devmapper/device_setup.go @@ -0,0 +1,236 @@ +package devmapper + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type directLVMConfig struct { + Device string + ThinpPercent uint64 + ThinpMetaPercent uint64 + AutoExtendPercent uint64 + AutoExtendThreshold uint64 +} + +var ( + errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified") + errThinpPercentTooBig = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100") + errMissingSetupDevice = errors.New("must provide device path in `dm.setup_device` in order to configure direct-lvm") +) + +func validateLVMConfig(cfg directLVMConfig) error { + if reflect.DeepEqual(cfg, directLVMConfig{}) { + return nil + } + if cfg.Device == "" { + return errMissingSetupDevice + } + if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 { + return errThinpPercentMissing + } + + if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 { + return errThinpPercentTooBig + } + return nil +} + +func checkDevAvailable(dev string) error { + lvmScan, err := exec.LookPath("lvmdiskscan") + if err != nil { + logrus.Debug("could not find lvmdiskscan") + return nil + } + + out, err := exec.Command(lvmScan).CombinedOutput() + if err != nil { + logrus.WithError(err).Error(string(out)) + return nil + } + + if !bytes.Contains(out, []byte(dev)) { + return errors.Errorf("%s is not available for use with devicemapper", dev) + } + return nil +} + +func checkDevInVG(dev string) error { + pvDisplay, err := exec.LookPath("pvdisplay") + if err != nil { + logrus.Debug("could not find pvdisplay") + return nil + } + + out, err := exec.Command(pvDisplay, dev).CombinedOutput() + if err != nil { + logrus.WithError(err).Error(string(out)) + return nil + } + + scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out))) + for scanner.Scan() { + fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name") + if len(fields) > 1 { + // got "VG Name" line" + vg := strings.TrimSpace(fields[1]) + if len(vg) > 0 { + return errors.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg) + } + logrus.Error(fields) + break + } + } + return nil +} + +func checkDevHasFS(dev string) error { + blkid, err := exec.LookPath("blkid") + if err != nil { + logrus.Debug("could not find blkid") + return nil + } + + out, err := exec.Command(blkid, dev).CombinedOutput() + if err != nil { + logrus.WithError(err).Error(string(out)) + return nil + } + + fields := bytes.Fields(out) + for _, f := range fields { + kv := bytes.Split(f, []byte{'='}) + if bytes.Equal(kv[0], []byte("TYPE")) { + v := bytes.Trim(kv[1], "\"") + if len(v) > 0 { + return errors.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev) + } + return nil + } + } + return nil +} + +func verifyBlockDevice(dev string, force bool) error { + if err := checkDevAvailable(dev); err != nil { + return err + } + if err := checkDevInVG(dev); err != nil { + return err + } + + if force { + return nil + } + + if err := checkDevHasFS(dev); err != nil { + return err + } + return nil +} + +func readLVMConfig(root string) (directLVMConfig, error) { + var cfg directLVMConfig + + p := filepath.Join(root, "setup-config.json") + b, err := ioutil.ReadFile(p) + if err != nil { + if os.IsNotExist(err) { + return cfg, nil + } + return cfg, errors.Wrap(err, "error reading existing setup config") + } + + // check if this is just an empty file, no need to produce a json error later if so + if len(b) == 0 { + return cfg, nil + } + + err = json.Unmarshal(b, &cfg) + return cfg, errors.Wrap(err, "error unmarshaling previous device setup config") +} + +func writeLVMConfig(root string, cfg directLVMConfig) error { + p := filepath.Join(root, "setup-config.json") + b, err := json.Marshal(cfg) + if err != nil { + return errors.Wrap(err, "error marshalling direct lvm config") + } + err = ioutil.WriteFile(p, b, 0600) + return errors.Wrap(err, "error writing direct lvm config to file") +} + +func setupDirectLVM(cfg directLVMConfig) error { + lvmProfileDir := "/etc/lvm/profile" + binaries := []string{"pvcreate", "vgcreate", "lvcreate", "lvconvert", "lvchange", "thin_check"} + + for _, bin := range binaries { + if _, err := exec.LookPath(bin); err != nil { + return errors.Wrap(err, "error looking up command `"+bin+"` while setting up direct lvm") + } + } + + err := os.MkdirAll(lvmProfileDir, 0755) + if err != nil { + return errors.Wrap(err, "error creating lvm profile directory") + } + + if cfg.AutoExtendPercent == 0 { + cfg.AutoExtendPercent = 20 + } + + if cfg.AutoExtendThreshold == 0 { + cfg.AutoExtendThreshold = 80 + } + + if cfg.ThinpPercent == 0 { + cfg.ThinpPercent = 95 + } + if cfg.ThinpMetaPercent == 0 { + cfg.ThinpMetaPercent = 1 + } + + out, err := exec.Command("pvcreate", "-f", cfg.Device).CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + + out, err = exec.Command("vgcreate", "storage", cfg.Device).CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + + out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpool", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpoolmeta", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + + out, err = exec.Command("lvconvert", "-y", "--zero", "n", "-c", "512K", "--thinpool", "storage/thinpool", "--poolmetadata", "storage/thinpoolmeta").CombinedOutput() + if err != nil { + return errors.Wrap(err, string(out)) + } + + profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent) + err = ioutil.WriteFile(lvmProfileDir+"/storage-thinpool.profile", []byte(profile), 0600) + if err != nil { + return errors.Wrap(err, "error writing storage thinp autoextend profile") + } + + out, err = exec.Command("lvchange", "--metadataprofile", "storage-thinpool", "storage/thinpool").CombinedOutput() + return errors.Wrap(err, string(out)) +} diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index 2608c49f..6db7b2b2 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -12,44 +12,41 @@ import ( "os/exec" "path" "path/filepath" + "reflect" "strconv" "strings" "sync" - "syscall" "time" - "github.com/sirupsen/logrus" - "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/devicemapper" + "github.com/containers/storage/pkg/dmesg" "github.com/containers/storage/pkg/idtools" - "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/loopback" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" - "github.com/docker/go-units" - + "github.com/containers/storage/pkg/parsers/kernel" + units "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) var ( - defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 - defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 - defaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 - defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors - defaultUdevSyncOverride = false - maxDeviceID = 0xffffff // 24 bit, pool limit - deviceIDMapSz = (maxDeviceID + 1) / 8 - // We retry device removal so many a times that even error messages - // will fill up console during normal operation. So only log Fatal - // messages by default. - logLevel = devicemapper.LogLevelFatal + defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 + defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 + defaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 + defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors + defaultUdevSyncOverride = false + maxDeviceID = 0xffffff // 24 bit, pool limit + deviceIDMapSz = (maxDeviceID + 1) / 8 driverDeferredRemovalSupport = false enableDeferredRemoval = false enableDeferredDeletion = false userBaseSize = false defaultMinFreeSpacePercent uint32 = 10 + lvmSetupConfigForce bool ) const deviceSetMetaFile string = "deviceset-metadata" @@ -122,6 +119,8 @@ type DeviceSet struct { uidMaps []idtools.IDMap gidMaps []idtools.IDMap minFreeSpacePercent uint32 //min free space percentage in thinpool + xfsNospaceRetries string // max retries when xfs receives ENOSPC + lvmSetupConfig directLVMConfig } // DiskUsage contains information about disk usage and is used when reporting Status of a device. @@ -170,7 +169,7 @@ type Status struct { MinFreeSpace uint64 } -// Structure used to export image/container metadata in docker inspect. +// Structure used to export image/container metadata in inspect. type deviceMetadata struct { deviceID int deviceSize uint64 // size in bytes @@ -379,10 +378,7 @@ func (devices *DeviceSet) isDeviceIDFree(deviceID int) bool { var mask byte i := deviceID % 8 mask = (1 << uint(i)) - if (devices.deviceIDMap[deviceID/8] & mask) != 0 { - return false - } - return true + return (devices.deviceIDMap[deviceID/8] & mask) == 0 } // Should be called with devices.Lock() held. @@ -409,8 +405,8 @@ func (devices *DeviceSet) lookupDeviceWithLock(hash string) (*devInfo, error) { // This function relies on that device hash map has been loaded in advance. // Should be called with devices.Lock() held. func (devices *DeviceSet) constructDeviceIDMap() { - logrus.Debugf("devmapper: constructDeviceIDMap()") - defer logrus.Debugf("devmapper: constructDeviceIDMap() END") + logrus.Debug("devmapper: constructDeviceIDMap()") + defer logrus.Debug("devmapper: constructDeviceIDMap() END") for _, info := range devices.Devices { devices.markDeviceIDUsed(info.DeviceID) @@ -458,8 +454,8 @@ func (devices *DeviceSet) deviceFileWalkFunction(path string, finfo os.FileInfo) } func (devices *DeviceSet) loadDeviceFilesOnStart() error { - logrus.Debugf("devmapper: loadDeviceFilesOnStart()") - defer logrus.Debugf("devmapper: loadDeviceFilesOnStart() END") + logrus.Debug("devmapper: loadDeviceFilesOnStart()") + defer logrus.Debug("devmapper: loadDeviceFilesOnStart() END") var scan = func(path string, info os.FileInfo, err error) error { if err != nil { @@ -479,11 +475,10 @@ func (devices *DeviceSet) loadDeviceFilesOnStart() error { } // Should be called with devices.Lock() held. -func (devices *DeviceSet) unregisterDevice(id int, hash string) error { - logrus.Debugf("devmapper: unregisterDevice(%v, %v)", id, hash) +func (devices *DeviceSet) unregisterDevice(hash string) error { + logrus.Debugf("devmapper: unregisterDevice(%v)", hash) info := &devInfo{ - Hash: hash, - DeviceID: id, + Hash: hash, } delete(devices.Devices, hash) @@ -528,7 +523,7 @@ func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bo // Make sure deferred removal on device is canceled, if one was // scheduled. - if err := devices.cancelDeferredRemoval(info); err != nil { + if err := devices.cancelDeferredRemovalIfNeeded(info); err != nil { return fmt.Errorf("devmapper: Device Deferred Removal Cancellation Failed: %s", err) } @@ -539,11 +534,11 @@ func (devices *DeviceSet) activateDeviceIfNeeded(info *devInfo, ignoreDeleted bo return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceID, info.Size) } -// Return true only if kernel supports xfs and mkfs.xfs is available -func xfsSupported() bool { +// xfsSupported checks if xfs is supported, returns nil if it is, otherwise an error +func xfsSupported() error { // Make sure mkfs.xfs is available if _, err := exec.LookPath("mkfs.xfs"); err != nil { - return false + return err // error text is descriptive enough } // Check if kernel supports xfs filesystem or not. @@ -551,43 +546,48 @@ func xfsSupported() bool { f, err := os.Open("/proc/filesystems") if err != nil { - logrus.Warnf("devmapper: Could not check if xfs is supported: %v", err) - return false + return errors.Wrapf(err, "error checking for xfs support") } defer f.Close() s := bufio.NewScanner(f) for s.Scan() { if strings.HasSuffix(s.Text(), "\txfs") { - return true + return nil } } if err := s.Err(); err != nil { - logrus.Warnf("devmapper: Could not check if xfs is supported: %v", err) + return errors.Wrapf(err, "error checking for xfs support") } - return false + + return errors.New(`kernel does not support xfs, or "modprobe xfs" failed`) } func determineDefaultFS() string { - if xfsSupported() { + err := xfsSupported() + if err == nil { return "xfs" } - logrus.Warn("devmapper: XFS is not supported in your system. Either the kernel doesn't support it or mkfs.xfs is not in your PATH. Defaulting to ext4 filesystem") + logrus.Warnf("devmapper: XFS is not supported in your system (%v). Defaulting to ext4 filesystem", err) return "ext4" } +// mkfsOptions tries to figure out whether some additional mkfs options are required +func mkfsOptions(fs string) []string { + if fs == "xfs" && !kernel.CheckKernelVersion(3, 16, 0) { + // For kernels earlier than 3.16 (and newer xfsutils), + // some xfs features need to be explicitly disabled. + return []string{"-m", "crc=0,finobt=0"} + } + + return []string{} +} + func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) { devname := info.DevName() - args := []string{} - for _, arg := range devices.mkfsArgs { - args = append(args, arg) - } - - args = append(args, devname) - if devices.filesystem == "" { devices.filesystem = determineDefaultFS() } @@ -595,7 +595,11 @@ func (devices *DeviceSet) createFilesystem(info *devInfo) (err error) { return err } - logrus.Infof("devmapper: Creating filesystem %s on device %s", devices.filesystem, info.Name()) + args := mkfsOptions(devices.filesystem) + args = append(args, devices.mkfsArgs...) + args = append(args, devname) + + logrus.Infof("devmapper: Creating filesystem %s on device %s, mkfs args: %v", devices.filesystem, info.Name(), args) defer func() { if err != nil { logrus.Infof("devmapper: Error while creating filesystem %s on device %s: %v", devices.filesystem, info.Name(), err) @@ -833,7 +837,7 @@ func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) { } if err := devices.closeTransaction(); err != nil { - devices.unregisterDevice(deviceID, hash) + devices.unregisterDevice(hash) devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) devices.markDeviceIDFree(deviceID) return nil, err @@ -841,11 +845,57 @@ func (devices *DeviceSet) createRegisterDevice(hash string) (*devInfo, error) { return info, nil } -func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo, size uint64) error { - if err := devices.poolHasFreeSpace(); err != nil { +func (devices *DeviceSet) takeSnapshot(hash string, baseInfo *devInfo, size uint64) error { + var ( + devinfo *devicemapper.Info + err error + ) + + if err = devices.poolHasFreeSpace(); err != nil { return err } + if devices.deferredRemove { + devinfo, err = devicemapper.GetInfoWithDeferred(baseInfo.Name()) + if err != nil { + return err + } + if devinfo != nil && devinfo.DeferredRemove != 0 { + err = devices.cancelDeferredRemoval(baseInfo) + if err != nil { + // If Error is ErrEnxio. Device is probably already gone. Continue. + if errors.Cause(err) != devicemapper.ErrEnxio { + return err + } + devinfo = nil + } else { + defer devices.deactivateDevice(baseInfo) + } + } + } else { + devinfo, err = devicemapper.GetInfo(baseInfo.Name()) + if err != nil { + return err + } + } + + doSuspend := devinfo != nil && devinfo.Exists != 0 + + if doSuspend { + if err = devicemapper.SuspendDevice(baseInfo.Name()); err != nil { + return err + } + defer devicemapper.ResumeDevice(baseInfo.Name()) + } + + if err = devices.createRegisterSnapDevice(hash, baseInfo, size); err != nil { + return err + } + + return nil +} + +func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInfo, size uint64) error { deviceID, err := devices.getNextFreeDeviceID() if err != nil { return err @@ -858,7 +908,7 @@ func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInf } for { - if err := devicemapper.CreateSnapDevice(devices.getPoolDevName(), deviceID, baseInfo.Name(), baseInfo.DeviceID); err != nil { + if err := devicemapper.CreateSnapDeviceRaw(devices.getPoolDevName(), deviceID, baseInfo.DeviceID); err != nil { if devicemapper.DeviceIDExists(err) { // Device ID already exists. This should not // happen. Now we have a mechanism to find @@ -888,7 +938,7 @@ func (devices *DeviceSet) createRegisterSnapDevice(hash string, baseInfo *devInf } if err := devices.closeTransaction(); err != nil { - devices.unregisterDevice(deviceID, hash) + devices.unregisterDevice(hash) devicemapper.DeleteDevice(devices.getPoolDevName(), deviceID) devices.markDeviceIDFree(deviceID) return err @@ -1134,7 +1184,7 @@ func (devices *DeviceSet) growFS(info *devInfo) error { defer devices.deactivateDevice(info) - fsMountPoint := "/run/containers/mnt" + fsMountPoint := "/run/containers/storage/mnt" if _, err := os.Stat(fsMountPoint); os.IsNotExist(err) { if err := os.MkdirAll(fsMountPoint, 0700); err != nil { return err @@ -1150,10 +1200,10 @@ func (devices *DeviceSet) growFS(info *devInfo) error { options = joinMountOptions(options, devices.mountOptions) if err := mount.Mount(info.DevName(), fsMountPoint, devices.BaseDeviceFilesystem, options); err != nil { - return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), fsMountPoint, err) + return fmt.Errorf("Error mounting '%s' on '%s': %s\n%v", info.DevName(), fsMountPoint, err, string(dmesg.Dmesg(256))) } - defer syscall.Unmount(fsMountPoint, syscall.MNT_DETACH) + defer unix.Unmount(fsMountPoint, unix.MNT_DETACH) switch devices.BaseDeviceFilesystem { case "ext4": @@ -1216,39 +1266,18 @@ func (devices *DeviceSet) setupBaseImage() error { } func setCloseOnExec(name string) { - if fileInfos, _ := ioutil.ReadDir("/proc/self/fd"); fileInfos != nil { - for _, i := range fileInfos { - link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) - if link == name { - fd, err := strconv.Atoi(i.Name()) - if err == nil { - syscall.CloseOnExec(fd) - } + fileInfos, _ := ioutil.ReadDir("/proc/self/fd") + for _, i := range fileInfos { + link, _ := os.Readlink(filepath.Join("/proc/self/fd", i.Name())) + if link == name { + fd, err := strconv.Atoi(i.Name()) + if err == nil { + unix.CloseOnExec(fd) } } } } -// DMLog implements logging using DevMapperLogger interface. -func (devices *DeviceSet) DMLog(level int, file string, line int, dmError int, message string) { - // By default libdm sends us all the messages including debug ones. - // We need to filter out messages here and figure out which one - // should be printed. - if level > logLevel { - return - } - - // FIXME(vbatts) push this back into ./pkg/devicemapper/ - if level <= devicemapper.LogLevelErr { - logrus.Errorf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) - } else if level <= devicemapper.LogLevelInfo { - logrus.Infof("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) - } else { - // FIXME(vbatts) push this back into ./pkg/devicemapper/ - logrus.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) - } -} - func major(device uint64) uint64 { return (device >> 8) & 0xfff } @@ -1356,10 +1385,7 @@ func (devices *DeviceSet) saveTransactionMetaData() error { } func (devices *DeviceSet) removeTransactionMetaData() error { - if err := os.RemoveAll(devices.transactionMetaFile()); err != nil { - return err - } - return nil + return os.RemoveAll(devices.transactionMetaFile()) } func (devices *DeviceSet) rollbackTransaction() error { @@ -1464,12 +1490,9 @@ func (devices *DeviceSet) closeTransaction() error { } func determineDriverCapabilities(version string) error { - /* - * Driver version 4.27.0 and greater support deferred activation - * feature. - */ + // Kernel driver version >= 4.27.0 support deferred removal - logrus.Debugf("devicemapper: driver version is %s", version) + logrus.Debugf("devicemapper: kernel dm driver version is %s", version) versionSplit := strings.Split(version, ".") major, err := strconv.Atoi(versionSplit[0]) @@ -1505,12 +1528,13 @@ func determineDriverCapabilities(version string) error { // Determine the major and minor number of loopback device func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) { - stat, err := file.Stat() + var stat unix.Stat_t + err := unix.Stat(file.Name(), &stat) if err != nil { return 0, 0, err } - dev := stat.Sys().(*syscall.Stat_t).Rdev + dev := stat.Rdev majorNum := major(dev) minorNum := minor(dev) @@ -1648,36 +1672,19 @@ func (devices *DeviceSet) enableDeferredRemovalDeletion() error { return nil } -func (devices *DeviceSet) initDevmapper(doInit bool) error { - // give ourselves to libdm as a log handler - devicemapper.LogInit(devices) - - version, err := devicemapper.GetDriverVersion() - if err != nil { - // Can't even get driver version, assume not supported - return errors.Wrap(graphdriver.ErrNotSupported, "unable to determine version of device mapper") - } - - if err := determineDriverCapabilities(version); err != nil { - return errors.Wrap(graphdriver.ErrNotSupported, "unable to determine device mapper driver capabilities") - } - +func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) { if err := devices.enableDeferredRemovalDeletion(); err != nil { return err } // https://github.com/docker/docker/issues/4036 - // if supported := devicemapper.UdevSetSyncSupport(true); !supported { - // if storageversion.IAmStatic == "true" { - // logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option") - // } else { - // logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option") - // } - // - // if !devices.overrideUdevSyncCheck { - // return graphdriver.ErrNotSupported - // } - // } + if supported := devicemapper.UdevSetSyncSupport(true); !supported { + logrus.Error("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/dockerd/#storage-driver-options") + + if !devices.overrideUdevSyncCheck { + return graphdriver.ErrNotSupported + } + } //create the root dir of the devmapper driver ownership to match this //daemon's remapped root uid/gid so containers can start properly @@ -1692,20 +1699,47 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { return err } - // Set the device prefix from the device id and inode of the container root dir - - st, err := os.Stat(devices.root) + prevSetupConfig, err := readLVMConfig(devices.root) if err != nil { + return err + } + + if !reflect.DeepEqual(devices.lvmSetupConfig, directLVMConfig{}) { + if devices.thinPoolDevice != "" { + return errors.New("cannot setup direct-lvm when `dm.thinpooldev` is also specified") + } + + if !reflect.DeepEqual(prevSetupConfig, devices.lvmSetupConfig) { + if !reflect.DeepEqual(prevSetupConfig, directLVMConfig{}) { + return errors.New("changing direct-lvm config is not supported") + } + logrus.WithField("storage-driver", "devicemapper").WithField("direct-lvm-config", devices.lvmSetupConfig).Debugf("Setting up direct lvm mode") + if err := verifyBlockDevice(devices.lvmSetupConfig.Device, lvmSetupConfigForce); err != nil { + return err + } + if err := setupDirectLVM(devices.lvmSetupConfig); err != nil { + return err + } + if err := writeLVMConfig(devices.root, devices.lvmSetupConfig); err != nil { + return err + } + } + devices.thinPoolDevice = "storage-thinpool" + logrus.WithField("storage-driver", "devicemapper").Debugf("Setting dm.thinpooldev to %q", devices.thinPoolDevice) + } + + // Set the device prefix from the device id and inode of the storage root dir + var st unix.Stat_t + if err := unix.Stat(devices.root, &st); err != nil { return fmt.Errorf("devmapper: Error looking up dir %s: %s", devices.root, err) } - sysSt := st.Sys().(*syscall.Stat_t) // "reg-" stands for "regular file". // In the future we might use "dev-" for "device file", etc. // container-maj,min[-inode] stands for: // - Managed by container storage // - The target of this device is at major and minor // - If is defined, use that file inside the device as a loopback image. Otherwise use the device itself. - devices.devicePrefix = fmt.Sprintf("container-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino) + devices.devicePrefix = fmt.Sprintf("container-%d:%d-%d", major(st.Dev), minor(st.Dev), st.Ino) logrus.Debugf("devmapper: Generated prefix: %s", devices.devicePrefix) // Check for the existence of the thin-pool device @@ -1748,7 +1782,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { hasData := devices.hasImage("data") if !doInit && !hasData { - return errors.New("Loopback data file not found") + return errors.New("loopback data file not found") } if !hasData { @@ -1781,7 +1815,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { hasMetadata := devices.hasImage("metadata") if !doInit && !hasMetadata { - return errors.New("Loopback metadata file not found") + return errors.New("loopback metadata file not found") } if !hasMetadata { @@ -1811,6 +1845,14 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { return err } + defer func() { + if retErr != nil { + err = devices.deactivatePool() + if err != nil { + logrus.Warnf("devmapper: Failed to deactivatePool: %v", err) + } + } + }() } // Pool already exists and caller did not pass us a pool. That means @@ -1857,8 +1899,8 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { // AddDevice adds a device and registers in the hash. func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string]string) error { - logrus.Debugf("devmapper: AddDevice(hash=%s basehash=%s)", hash, baseHash) - defer logrus.Debugf("devmapper: AddDevice(hash=%s basehash=%s) END", hash, baseHash) + logrus.Debugf("devmapper: AddDevice START(hash=%s basehash=%s)", hash, baseHash) + defer logrus.Debugf("devmapper: AddDevice END(hash=%s basehash=%s)", hash, baseHash) // If a deleted device exists, return error. baseInfo, err := devices.lookupDeviceWithLock(baseHash) @@ -1895,7 +1937,7 @@ func (devices *DeviceSet) AddDevice(hash, baseHash string, storageOpt map[string return fmt.Errorf("devmapper: Container size cannot be smaller than %s", units.HumanSize(float64(baseInfo.Size))) } - if err := devices.createRegisterSnapDevice(hash, baseInfo, size); err != nil { + if err := devices.takeSnapshot(hash, baseInfo, size); err != nil { return err } @@ -1975,7 +2017,7 @@ func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) erro } if err == nil { - if err := devices.unregisterDevice(info.DeviceID, info.Hash); err != nil { + if err := devices.unregisterDevice(info.Hash); err != nil { return err } // If device was already in deferred delete state that means @@ -1996,8 +2038,8 @@ func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) erro // Issue discard only if device open count is zero. func (devices *DeviceSet) issueDiscard(info *devInfo) error { - logrus.Debugf("devmapper: issueDiscard(device: %s). START", info.Hash) - defer logrus.Debugf("devmapper: issueDiscard(device: %s). END", info.Hash) + logrus.Debugf("devmapper: issueDiscard START(device: %s).", info.Hash) + defer logrus.Debugf("devmapper: issueDiscard END(device: %s).", info.Hash) // This is a workaround for the kernel not discarding block so // on the thin pool when we remove a thinp device, so we do it // manually. @@ -2030,7 +2072,16 @@ func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error { } // Try to deactivate device in case it is active. - if err := devices.deactivateDevice(info); err != nil { + // If deferred removal is enabled and deferred deletion is disabled + // then make sure device is removed synchronously. There have been + // some cases of device being busy for short duration and we would + // rather busy wait for device removal to take care of these cases. + deferredRemove := devices.deferredRemove + if !devices.deferredDelete { + deferredRemove = false + } + + if err := devices.deactivateDeviceMode(info, deferredRemove); err != nil { logrus.Debugf("devmapper: Error deactivating device: %s", err) return err } @@ -2046,8 +2097,8 @@ func (devices *DeviceSet) deleteDevice(info *devInfo, syncDelete bool) error { // removal. If one wants to override that and want DeleteDevice() to fail if // device was busy and could not be deleted, set syncDelete=true. func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error { - logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) START", hash, syncDelete) - defer logrus.Debugf("devmapper: DeleteDevice(hash=%v syncDelete=%v) END", hash, syncDelete) + logrus.Debugf("devmapper: DeleteDevice START(hash=%v syncDelete=%v)", hash, syncDelete) + defer logrus.Debugf("devmapper: DeleteDevice END(hash=%v syncDelete=%v)", hash, syncDelete) info, err := devices.lookupDeviceWithLock(hash) if err != nil { return err @@ -2063,8 +2114,8 @@ func (devices *DeviceSet) DeleteDevice(hash string, syncDelete bool) error { } func (devices *DeviceSet) deactivatePool() error { - logrus.Debug("devmapper: deactivatePool()") - defer logrus.Debug("devmapper: deactivatePool END") + logrus.Debug("devmapper: deactivatePool() START") + defer logrus.Debug("devmapper: deactivatePool() END") devname := devices.getPoolDevName() devinfo, err := devicemapper.GetInfo(devname) @@ -2087,7 +2138,12 @@ func (devices *DeviceSet) deactivatePool() error { } func (devices *DeviceSet) deactivateDevice(info *devInfo) error { - logrus.Debugf("devmapper: deactivateDevice(%s)", info.Hash) + return devices.deactivateDeviceMode(info, devices.deferredRemove) +} + +func (devices *DeviceSet) deactivateDeviceMode(info *devInfo, deferredRemove bool) error { + var err error + logrus.Debugf("devmapper: deactivateDevice START(%s)", info.Hash) defer logrus.Debugf("devmapper: deactivateDevice END(%s)", info.Hash) devinfo, err := devicemapper.GetInfo(info.Name()) @@ -2099,14 +2155,17 @@ func (devices *DeviceSet) deactivateDevice(info *devInfo) error { return nil } - if devices.deferredRemove { - if err := devicemapper.RemoveDeviceDeferred(info.Name()); err != nil { - return err - } + if deferredRemove { + err = devicemapper.RemoveDeviceDeferred(info.Name()) } else { - if err := devices.removeDevice(info.Name()); err != nil { - return err - } + err = devices.removeDevice(info.Name()) + } + + // This function's semantics is such that it does not return an + // error if device does not exist. So if device went away by + // the time we actually tried to remove it, do not return error. + if errors.Cause(err) != devicemapper.ErrEnxio { + return err } return nil } @@ -2137,41 +2196,53 @@ func (devices *DeviceSet) removeDevice(devname string) error { return err } -func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error { +func (devices *DeviceSet) cancelDeferredRemovalIfNeeded(info *devInfo) error { if !devices.deferredRemove { return nil } - logrus.Debugf("devmapper: cancelDeferredRemoval START(%s)", info.Name()) - defer logrus.Debugf("devmapper: cancelDeferredRemoval END(%s)", info.Name()) + logrus.Debugf("devmapper: cancelDeferredRemovalIfNeeded START(%s)", info.Name()) + defer logrus.Debugf("devmapper: cancelDeferredRemovalIfNeeded END(%s)", info.Name()) devinfo, err := devicemapper.GetInfoWithDeferred(info.Name()) + if err != nil { + return err + } if devinfo != nil && devinfo.DeferredRemove == 0 { return nil } // Cancel deferred remove - for i := 0; i < 100; i++ { - err = devicemapper.CancelDeferredRemove(info.Name()) - if err == nil { - break - } - - if errors.Cause(err) == devicemapper.ErrEnxio { - // Device is probably already gone. Return success. - return nil - } - + if err := devices.cancelDeferredRemoval(info); err != nil { + // If Error is ErrEnxio. Device is probably already gone. Continue. if errors.Cause(err) != devicemapper.ErrBusy { return err } + } + return nil +} - // If we see EBUSY it may be a transient error, - // sleep a bit a retry a few times. - devices.Unlock() - time.Sleep(100 * time.Millisecond) - devices.Lock() +func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error { + logrus.Debugf("devmapper: cancelDeferredRemoval START(%s)", info.Name()) + defer logrus.Debugf("devmapper: cancelDeferredRemoval END(%s)", info.Name()) + + var err error + + // Cancel deferred remove + for i := 0; i < 100; i++ { + err = devicemapper.CancelDeferredRemove(info.Name()) + if err != nil { + if errors.Cause(err) != devicemapper.ErrBusy { + // If we see EBUSY it may be a transient error, + // sleep a bit a retry a few times. + devices.Unlock() + time.Sleep(100 * time.Millisecond) + devices.Lock() + continue + } + } + break } return err } @@ -2209,9 +2280,6 @@ func (devices *DeviceSet) Shutdown(home string) error { if err != nil { return err } - if p == path.Join(home, "mnt") { - return nil - } if !info.IsDir() { return nil } @@ -2220,7 +2288,7 @@ func (devices *DeviceSet) Shutdown(home string) error { // We use MNT_DETACH here in case it is still busy in some running // container. This means it'll go away from the global scope directly, // and the device will be released when that container dies. - if err := syscall.Unmount(p, syscall.MNT_DETACH); err != nil { + if err := unix.Unmount(p, unix.MNT_DETACH); err != nil { logrus.Debugf("devmapper: Shutdown unmounting %s, error: %s", p, err) } } @@ -2263,6 +2331,34 @@ func (devices *DeviceSet) Shutdown(home string) error { return nil } +// Recent XFS changes allow changing behavior of filesystem in case of errors. +// When thin pool gets full and XFS gets ENOSPC error, currently it tries +// IO infinitely and sometimes it can block the container process +// and process can't be killWith 0 value, XFS will not retry upon error +// and instead will shutdown filesystem. + +func (devices *DeviceSet) xfsSetNospaceRetries(info *devInfo) error { + dmDevicePath, err := os.Readlink(info.DevName()) + if err != nil { + return fmt.Errorf("devmapper: readlink failed for device %v:%v", info.DevName(), err) + } + + dmDeviceName := path.Base(dmDevicePath) + filePath := "/sys/fs/xfs/" + dmDeviceName + "/error/metadata/ENOSPC/max_retries" + maxRetriesFile, err := os.OpenFile(filePath, os.O_WRONLY, 0) + if err != nil { + return fmt.Errorf("devmapper: user specified daemon option dm.xfs_nospace_max_retries but it does not seem to be supported on this system :%v", err) + } + defer maxRetriesFile.Close() + + // Set max retries to 0 + _, err = maxRetriesFile.WriteString(devices.xfsNospaceRetries) + if err != nil { + return fmt.Errorf("devmapper: Failed to write string %v to file %v:%v", devices.xfsNospaceRetries, filePath, err) + } + return nil +} + // MountDevice mounts the device if not already mounted. func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { info, err := devices.lookupDeviceWithLock(hash) @@ -2300,7 +2396,15 @@ func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { options = joinMountOptions(options, label.FormatMountLabel("", mountLabel)) if err := mount.Mount(info.DevName(), path, fstype, options); err != nil { - return fmt.Errorf("devmapper: Error mounting '%s' on '%s': %s", info.DevName(), path, err) + return fmt.Errorf("devmapper: Error mounting '%s' on '%s': %s\n%v", info.DevName(), path, err, string(dmesg.Dmesg(256))) + } + + if fstype == "xfs" && devices.xfsNospaceRetries != "" { + if err := devices.xfsSetNospaceRetries(info); err != nil { + unix.Unmount(path, unix.MNT_DETACH) + devices.deactivateDevice(info) + return err + } } return nil @@ -2308,8 +2412,8 @@ func (devices *DeviceSet) MountDevice(hash, path, mountLabel string) error { // UnmountDevice unmounts the device and removes it from hash. func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error { - logrus.Debugf("devmapper: UnmountDevice(hash=%s)", hash) - defer logrus.Debugf("devmapper: UnmountDevice(hash=%s) END", hash) + logrus.Debugf("devmapper: UnmountDevice START(hash=%s)", hash) + defer logrus.Debugf("devmapper: UnmountDevice END(hash=%s)", hash) info, err := devices.lookupDeviceWithLock(hash) if err != nil { @@ -2323,16 +2427,12 @@ func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error { defer devices.Unlock() logrus.Debugf("devmapper: Unmount(%s)", mountPath) - if err := syscall.Unmount(mountPath, syscall.MNT_DETACH); err != nil { + if err := unix.Unmount(mountPath, unix.MNT_DETACH); err != nil { return err } logrus.Debug("devmapper: Unmount done") - if err := devices.deactivateDevice(info); err != nil { - return err - } - - return nil + return devices.deactivateDevice(info) } // HasDevice returns true if the device metadata exists. @@ -2424,8 +2524,8 @@ func (devices *DeviceSet) MetadataDevicePath() string { } func (devices *DeviceSet) getUnderlyingAvailableSpace(loopFile string) (uint64, error) { - buf := new(syscall.Statfs_t) - if err := syscall.Statfs(loopFile, buf); err != nil { + buf := new(unix.Statfs_t) + if err := unix.Statfs(loopFile, buf); err != nil { logrus.Warnf("devmapper: Couldn't stat loopfile filesystem %v: %v", loopFile, err) return 0, err } @@ -2534,22 +2634,25 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ minFreeSpacePercent: defaultMinFreeSpacePercent, } - // Pick up initialization settings, if any were saved before - defaultsFile := path.Join(root, "defaults") - defaultsBytes, err := ioutil.ReadFile(defaultsFile) - defaults := []string{} - settings := map[string]string{} - if err == nil && len(defaultsBytes) > 0 { - defaults = strings.Split(string(defaultsBytes), "\n") + version, err := devicemapper.GetDriverVersion() + if err != nil { + // Can't even get driver version, assume not supported + return nil, graphdriver.ErrNotSupported + } + + if err := determineDriverCapabilities(version); err != nil { + return nil, graphdriver.ErrNotSupported + } + + if driverDeferredRemovalSupport && devicemapper.LibraryDeferredRemovalSupport { + // enable deferred stuff by default + enableDeferredDeletion = true + enableDeferredRemoval = true } foundBlkDiscard := false - nthOption := 0 - for _, option := range append(defaults, options...) { - nthOption = nthOption + 1 - if len(option) == 0 { - continue - } + var lvmSetupConfig directLVMConfig + for _, option := range options { key, val, err := parsers.ParseKeyValueOpt(option) if err != nil { return nil, err @@ -2637,15 +2740,78 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ } devices.minFreeSpacePercent = uint32(minFreeSpacePercent) - default: - if nthOption > len(defaults) { - return nil, fmt.Errorf("devmapper: Unknown option %s", key) + case "dm.xfs_nospace_max_retries": + _, err := strconv.ParseUint(val, 10, 64) + if err != nil { + return nil, err } - logrus.Errorf("devmapper: Unknown option %s, ignoring", key) + devices.xfsNospaceRetries = val + case "dm.directlvm_device": + lvmSetupConfig.Device = val + case "dm.directlvm_device_force": + lvmSetupConfigForce, err = strconv.ParseBool(val) + if err != nil { + return nil, err + } + case "dm.thinp_percent": + per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.thinp_percent=%s`", val) + } + if per >= 100 { + return nil, errors.New("dm.thinp_percent must be greater than 0 and less than 100") + } + lvmSetupConfig.ThinpPercent = per + case "dm.thinp_metapercent": + per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.thinp_metapercent=%s`", val) + } + if per >= 100 { + return nil, errors.New("dm.thinp_metapercent must be greater than 0 and less than 100") + } + lvmSetupConfig.ThinpMetaPercent = per + case "dm.thinp_autoextend_percent": + per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_percent=%s`", val) + } + if per > 100 { + return nil, errors.New("dm.thinp_autoextend_percent must be greater than 0 and less than 100") + } + lvmSetupConfig.AutoExtendPercent = per + case "dm.thinp_autoextend_threshold": + per, err := strconv.ParseUint(strings.TrimSuffix(val, "%"), 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.thinp_autoextend_threshold=%s`", val) + } + if per > 100 { + return nil, errors.New("dm.thinp_autoextend_threshold must be greater than 0 and less than 100") + } + lvmSetupConfig.AutoExtendThreshold = per + case "dm.libdm_log_level": + level, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return nil, errors.Wrapf(err, "could not parse `dm.libdm_log_level=%s`", val) + } + if level < devicemapper.LogLevelFatal || level > devicemapper.LogLevelDebug { + return nil, errors.Errorf("dm.libdm_log_level must be in range [%d,%d]", devicemapper.LogLevelFatal, devicemapper.LogLevelDebug) + } + // Register a new logging callback with the specified level. + devicemapper.LogInit(devicemapper.DefaultLogger{ + Level: int(level), + }) + default: + return nil, fmt.Errorf("devmapper: Unknown option %s", key) } - settings[key] = val } + if err := validateLVMConfig(lvmSetupConfig); err != nil { + return nil, err + } + + devices.lvmSetupConfig = lvmSetupConfig + // By default, don't do blk discard hack on raw devices, its rarely useful and is expensive if !foundBlkDiscard && (devices.dataDevice != "" || devices.thinPoolDevice != "") { devices.doBlkDiscard = false @@ -2655,15 +2821,5 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ return nil, err } - // Save these settings along with the other metadata - defaults = []string{} - for key, val := range settings { - defaults = append(defaults, key+"="+val) - } - defaultsBytes = []byte(strings.Join(defaults, "\n") + "\n") - if err := ioutils.AtomicWriteFile(defaultsFile, defaultsBytes, 0600); err != nil { - return nil, err - } - return devices, nil } diff --git a/vendor/github.com/containers/storage/drivers/devmapper/driver.go b/vendor/github.com/containers/storage/drivers/devmapper/driver.go index 87a427a8..d68fb66c 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/driver.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/driver.go @@ -14,8 +14,10 @@ import ( "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/devicemapper" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/locker" "github.com/containers/storage/pkg/mount" - "github.com/docker/go-units" + "github.com/containers/storage/pkg/system" + units "github.com/docker/go-units" ) func init() { @@ -29,6 +31,7 @@ type Driver struct { uidMaps []idtools.IDMap gidMaps []idtools.IDMap ctr *graphdriver.RefCounter + locker *locker.Locker } // Init creates a driver with the given home and the set of options. @@ -48,6 +51,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap uidMaps: uidMaps, gidMaps: gidMaps, ctr: graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()), + locker: locker.New(), } return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil @@ -65,18 +69,18 @@ func (d *Driver) Status() [][2]string { status := [][2]string{ {"Pool Name", s.PoolName}, - {"Pool Blocksize", fmt.Sprintf("%s", units.HumanSize(float64(s.SectorSize)))}, - {"Base Device Size", fmt.Sprintf("%s", units.HumanSize(float64(s.BaseDeviceSize)))}, + {"Pool Blocksize", units.HumanSize(float64(s.SectorSize))}, + {"Base Device Size", units.HumanSize(float64(s.BaseDeviceSize))}, {"Backing Filesystem", s.BaseDeviceFS}, {"Data file", s.DataFile}, {"Metadata file", s.MetadataFile}, - {"Data Space Used", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Used)))}, - {"Data Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Total)))}, - {"Data Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Data.Available)))}, - {"Metadata Space Used", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Used)))}, - {"Metadata Space Total", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Total)))}, - {"Metadata Space Available", fmt.Sprintf("%s", units.HumanSize(float64(s.Metadata.Available)))}, - {"Thin Pool Minimum Free Space", fmt.Sprintf("%s", units.HumanSize(float64(s.MinFreeSpace)))}, + {"Data Space Used", units.HumanSize(float64(s.Data.Used))}, + {"Data Space Total", units.HumanSize(float64(s.Data.Total))}, + {"Data Space Available", units.HumanSize(float64(s.Data.Available))}, + {"Metadata Space Used", units.HumanSize(float64(s.Metadata.Used))}, + {"Metadata Space Total", units.HumanSize(float64(s.Metadata.Total))}, + {"Metadata Space Available", units.HumanSize(float64(s.Metadata.Available))}, + {"Thin Pool Minimum Free Space", units.HumanSize(float64(s.MinFreeSpace))}, {"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)}, {"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)}, {"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)}, @@ -122,12 +126,17 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return d.Create(id, parent, opts) } // Create adds a device with a given id and the parent. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + var storageOpt map[string]string + if opts != nil { + storageOpt = opts.StorageOpt + } + if err := d.DeviceSet.AddDevice(id, parent, storageOpt); err != nil { return err } @@ -137,6 +146,8 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str // Remove removes a device with a given id, unmounts the filesystem. func (d *Driver) Remove(id string) error { + d.locker.Lock(id) + defer d.locker.Unlock(id) if !d.DeviceSet.HasDevice(id) { // Consider removing a non-existing device a no-op // This is useful to be able to progress on container removal @@ -146,19 +157,15 @@ func (d *Driver) Remove(id string) error { // This assumes the device has been properly Get/Put:ed and thus is unmounted if err := d.DeviceSet.DeleteDevice(id, false); err != nil { - return err + return fmt.Errorf("failed to remove device %s: %v", id, err) } - - mp := path.Join(d.home, "mnt", id) - if err := os.RemoveAll(mp); err != nil && !os.IsNotExist(err) { - return err - } - - return nil + return system.EnsureRemoveAll(path.Join(d.home, "mnt", id)) } // Get mounts a device with given id into the root filesystem func (d *Driver) Get(id, mountLabel string) (string, error) { + d.locker.Lock(id) + defer d.locker.Unlock(id) mp := path.Join(d.home, "mnt", id) rootFs := path.Join(mp, "rootfs") if count := d.ctr.Increment(mp); count > 1 { @@ -209,6 +216,8 @@ func (d *Driver) Get(id, mountLabel string) (string, error) { // Put unmounts a device and removes it. func (d *Driver) Put(id string) error { + d.locker.Lock(id) + defer d.locker.Unlock(id) mp := path.Join(d.home, "mnt", id) if count := d.ctr.Decrement(mp); count > 0 { return nil @@ -227,6 +236,5 @@ func (d *Driver) Exists(id string) bool { // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { - var imageStores []string - return imageStores + return nil } diff --git a/vendor/github.com/containers/storage/drivers/devmapper/mount.go b/vendor/github.com/containers/storage/drivers/devmapper/mount.go index cca1fe1b..1dc3262d 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/mount.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/mount.go @@ -7,7 +7,8 @@ import ( "fmt" "os" "path/filepath" - "syscall" + + "golang.org/x/sys/unix" ) // FIXME: this is copy-pasted from the aufs driver. @@ -15,19 +16,17 @@ import ( // Mounted returns true if a mount point exists. func Mounted(mountpoint string) (bool, error) { - mntpoint, err := os.Stat(mountpoint) - if err != nil { + var mntpointSt unix.Stat_t + if err := unix.Stat(mountpoint, &mntpointSt); err != nil { if os.IsNotExist(err) { return false, nil } return false, err } - parent, err := os.Stat(filepath.Join(mountpoint, "..")) - if err != nil { + var parentSt unix.Stat_t + if err := unix.Stat(filepath.Join(mountpoint, ".."), &parentSt); err != nil { return false, err } - mntpointSt := mntpoint.Sys().(*syscall.Stat_t) - parentSt := parent.Sys().(*syscall.Stat_t) return mntpointSt.Dev != parentSt.Dev, nil } diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index cfa33209..56996478 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -29,12 +29,19 @@ var ( // ErrNotSupported returned when driver is not supported. ErrNotSupported = errors.New("driver not supported") - // ErrPrerequisites retuned when driver does not meet prerequisites. + // ErrPrerequisites returned when driver does not meet prerequisites. ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)") // ErrIncompatibleFS returned when file system is not supported. ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver") ) +//CreateOpts contains optional arguments for Create() and CreateReadWrite() +// methods. +type CreateOpts struct { + MountLabel string + StorageOpt map[string]string +} + // InitFunc initializes the storage driver. type InitFunc func(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) @@ -48,11 +55,13 @@ type ProtoDriver interface { // String returns a string representation of this driver. String() string // CreateReadWrite creates a new, empty filesystem layer that is ready - // to be used as the storage for a container. - CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error + // to be used as the storage for a container. Additional options can + // be passed in opts. parent may be "" and opts may be nil. + CreateReadWrite(id, parent string, opts *CreateOpts) error // Create creates a new, empty, filesystem layer with the - // specified id and parent and mountLabel. Parent and mountLabel may be "". - Create(id, parent, mountLabel string, storageOpt map[string]string) error + // specified id and parent and options passed in opts. Parent + // may be "" and opts may be nil. + Create(id, parent string, opts *CreateOpts) error // Remove attempts to remove the filesystem layer with this id. Remove(id string) error // Get returns the mountpoint for the layered filesystem referred @@ -79,9 +88,8 @@ type ProtoDriver interface { AdditionalImageStores() []string } -// Driver is the interface for layered/snapshot file system drivers. -type Driver interface { - ProtoDriver +// DiffDriver is the interface to use to implement graph diffs +type DiffDriver interface { // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". Diff(id, parent string) (io.ReadCloser, error) @@ -99,6 +107,29 @@ type Driver interface { DiffSize(id, parent string) (size int64, err error) } +// Driver is the interface for layered/snapshot file system drivers. +type Driver interface { + ProtoDriver + DiffDriver +} + +// Capabilities defines a list of capabilities a driver may implement. +// These capabilities are not required; however, they do determine how a +// graphdriver can be used. +type Capabilities struct { + // Flags that this driver is capable of reproducing exactly equivalent + // diffs for read-only layers. If set, clients can rely on the driver + // for consistent tar streams, and avoid extra processing to account + // for potential differences (eg: the layer store's use of tar-split). + ReproducesExactDiffs bool +} + +// CapabilityDriver is the interface for layered file system drivers that +// can report on their Capabilities. +type CapabilityDriver interface { + Capabilities() Capabilities +} + // DiffGetterDriver is the interface for layered file system drivers that // provide a specialized function for getting file contents for tar-split. type DiffGetterDriver interface { @@ -137,15 +168,13 @@ func Register(name string, initFunc InitFunc) error { } // GetDriver initializes and returns the registered driver -func GetDriver(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) { +func GetDriver(name string, config Options) (Driver, error) { if initFunc, exists := drivers[name]; exists { - return initFunc(filepath.Join(home, name), options, uidMaps, gidMaps) + return initFunc(filepath.Join(config.Root, name), config.DriverOptions, config.UIDMaps, config.GIDMaps) } - if pluginDriver, err := lookupPlugin(name, home, options); err == nil { - return pluginDriver, nil - } - logrus.Errorf("Failed to GetDriver graph %s %s", name, home) - return nil, errors.Wrapf(ErrNotSupported, "failed to GetDriver graph %s %s", name, home) + + logrus.Errorf("Failed to GetDriver graph %s %s", name, config.Root) + return nil, errors.Wrapf(ErrNotSupported, "failed to GetDriver graph %s %s", name, config.Root) } // getBuiltinDriver initializes and returns the registered driver, but does not try to load from plugins @@ -157,15 +186,24 @@ func getBuiltinDriver(name, home string, options []string, uidMaps, gidMaps []id return nil, errors.Wrapf(ErrNotSupported, "failed to built-in GetDriver graph %s %s", name, home) } +// Options is used to initialize a graphdriver +type Options struct { + Root string + DriverOptions []string + UIDMaps []idtools.IDMap + GIDMaps []idtools.IDMap + ExperimentalEnabled bool +} + // New creates the driver and initializes it at the specified root. -func New(root string, name string, options []string, uidMaps, gidMaps []idtools.IDMap) (Driver, error) { +func New(name string, config Options) (Driver, error) { if name != "" { logrus.Debugf("[graphdriver] trying provided driver %q", name) // so the logs show specified driver - return GetDriver(name, root, options, uidMaps, gidMaps) + return GetDriver(name, config) } // Guess for prior driver - driversMap := scanPriorDrivers(root) + driversMap := scanPriorDrivers(config.Root) for _, name := range priority { if name == "vfs" { // don't use vfs even if there is state present. @@ -174,13 +212,13 @@ func New(root string, name string, options []string, uidMaps, gidMaps []idtools. if _, prior := driversMap[name]; prior { // of the state found from prior drivers, check in order of our priority // which we would prefer - driver, err := getBuiltinDriver(name, root, options, uidMaps, gidMaps) + driver, err := getBuiltinDriver(name, config.Root, config.DriverOptions, config.UIDMaps, config.GIDMaps) if err != nil { // unlike below, we will return error here, because there is prior // state, and now it is no longer supported/prereq/compatible, so // something changed and needs attention. Otherwise the daemon's // images would just "disappear". - logrus.Errorf("[graphdriver] prior storage driver %q failed: %s", name, err) + logrus.Errorf("[graphdriver] prior storage driver %s failed: %s", name, err) return nil, err } @@ -192,17 +230,17 @@ func New(root string, name string, options []string, uidMaps, gidMaps []idtools. driversSlice = append(driversSlice, name) } - return nil, fmt.Errorf("%q contains several valid graphdrivers: %s; Please cleanup or explicitly choose storage driver (-s )", root, strings.Join(driversSlice, ", ")) + return nil, fmt.Errorf("%s contains several valid graphdrivers: %s; Please cleanup or explicitly choose storage driver (-s )", config.Root, strings.Join(driversSlice, ", ")) } - logrus.Infof("[graphdriver] using prior storage driver %q", name) + logrus.Infof("[graphdriver] using prior storage driver: %s", name) return driver, nil } } // Check for priority drivers first for _, name := range priority { - driver, err := getBuiltinDriver(name, root, options, uidMaps, gidMaps) + driver, err := getBuiltinDriver(name, config.Root, config.DriverOptions, config.UIDMaps, config.GIDMaps) if err != nil { if isDriverNotSupported(err) { continue @@ -214,7 +252,7 @@ func New(root string, name string, options []string, uidMaps, gidMaps []idtools. // Check all registered drivers if no priority driver is found for name, initFunc := range drivers { - driver, err := initFunc(filepath.Join(root, name), options, uidMaps, gidMaps) + driver, err := initFunc(filepath.Join(config.Root, name), config.DriverOptions, config.UIDMaps, config.GIDMaps) if err != nil { if isDriverNotSupported(err) { continue diff --git a/vendor/github.com/containers/storage/drivers/driver_freebsd.go b/vendor/github.com/containers/storage/drivers/driver_freebsd.go index 2891a84f..53394b73 100644 --- a/vendor/github.com/containers/storage/drivers/driver_freebsd.go +++ b/vendor/github.com/containers/storage/drivers/driver_freebsd.go @@ -1,6 +1,10 @@ package graphdriver -import "syscall" +import ( + "syscall" + + "golang.org/x/sys/unix" +) var ( // Slice of drivers that should be used in an order @@ -11,7 +15,7 @@ var ( // Mounted checks if the given path is mounted as the fs type func Mounted(fsType FsMagic, mountPath string) (bool, error) { - var buf syscall.Statfs_t + var buf unix.Statfs_t if err := syscall.Statfs(mountPath, &buf); err != nil { return false, err } diff --git a/vendor/github.com/containers/storage/drivers/driver_linux.go b/vendor/github.com/containers/storage/drivers/driver_linux.go index 6e1f2ee3..94f7270e 100644 --- a/vendor/github.com/containers/storage/drivers/driver_linux.go +++ b/vendor/github.com/containers/storage/drivers/driver_linux.go @@ -4,9 +4,9 @@ package graphdriver import ( "path/filepath" - "syscall" "github.com/containers/storage/pkg/mount" + "golang.org/x/sys/unix" ) const ( @@ -66,13 +66,14 @@ var ( FsMagicAufs: "aufs", FsMagicBtrfs: "btrfs", FsMagicCramfs: "cramfs", + FsMagicEcryptfs: "ecryptfs", FsMagicExtfs: "extfs", FsMagicF2fs: "f2fs", FsMagicGPFS: "gpfs", FsMagicJffs2Fs: "jffs2", FsMagicJfs: "jfs", FsMagicNfsFs: "nfs", - FsMagicOverlay: "overlay", + FsMagicOverlay: "overlayfs", FsMagicRAMFs: "ramfs", FsMagicReiserFs: "reiserfs", FsMagicSmbFs: "smb", @@ -87,14 +88,14 @@ var ( // GetFSMagic returns the filesystem id given the path. func GetFSMagic(rootpath string) (FsMagic, error) { - var buf syscall.Statfs_t - if err := syscall.Statfs(filepath.Dir(rootpath), &buf); err != nil { + var buf unix.Statfs_t + if err := unix.Statfs(filepath.Dir(rootpath), &buf); err != nil { return 0, err } return FsMagic(buf.Type), nil } -// NewFsChecker returns a checker configured for the provied FsMagic +// NewFsChecker returns a checker configured for the provided FsMagic func NewFsChecker(t FsMagic) Checker { return &fsChecker{ t: t, @@ -126,8 +127,8 @@ func (c *defaultChecker) IsMounted(path string) bool { // Mounted checks if the given path is mounted as the fs type func Mounted(fsType FsMagic, mountPath string) (bool, error) { - var buf syscall.Statfs_t - if err := syscall.Statfs(mountPath, &buf); err != nil { + var buf unix.Statfs_t + if err := unix.Statfs(mountPath, &buf); err != nil { return false, err } return FsMagic(buf.Type) == fsType, nil diff --git a/vendor/github.com/containers/storage/drivers/driver_solaris.go b/vendor/github.com/containers/storage/drivers/driver_solaris.go index cfbc26e8..174fa967 100644 --- a/vendor/github.com/containers/storage/drivers/driver_solaris.go +++ b/vendor/github.com/containers/storage/drivers/driver_solaris.go @@ -19,8 +19,8 @@ import ( "path/filepath" "unsafe" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" + "github.com/containers/storage/pkg/mount" + "github.com/sirupsen/logrus" ) const ( @@ -45,22 +45,52 @@ func GetFSMagic(rootpath string) (FsMagic, error) { return 0, nil } +type fsChecker struct { + t FsMagic +} + +func (c *fsChecker) IsMounted(path string) bool { + m, _ := Mounted(c.t, path) + return m +} + +// NewFsChecker returns a checker configured for the provided FsMagic +func NewFsChecker(t FsMagic) Checker { + return &fsChecker{ + t: t, + } +} + +// NewDefaultChecker returns a check that parses /proc/mountinfo to check +// if the specified path is mounted. +// No-op on Solaris. +func NewDefaultChecker() Checker { + return &defaultChecker{} +} + +type defaultChecker struct { +} + +func (c *defaultChecker) IsMounted(path string) bool { + m, _ := mount.Mounted(path) + return m +} + // Mounted checks if the given path is mounted as the fs type //Solaris supports only ZFS for now func Mounted(fsType FsMagic, mountPath string) (bool, error) { cs := C.CString(filepath.Dir(mountPath)) + defer C.free(unsafe.Pointer(cs)) buf := C.getstatfs(cs) + defer C.free(unsafe.Pointer(buf)) // on Solaris buf.f_basetype contains ['z', 'f', 's', 0 ... ] if (buf.f_basetype[0] != 122) || (buf.f_basetype[1] != 102) || (buf.f_basetype[2] != 115) || (buf.f_basetype[3] != 0) { - log.Debugf("[zfs] no zfs dataset found for rootdir '%s'", mountPath) - C.free(unsafe.Pointer(buf)) - return false, errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", mountPath) + logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", mountPath) + return false, ErrPrerequisites } - C.free(unsafe.Pointer(buf)) - C.free(unsafe.Pointer(cs)) return true, nil } diff --git a/vendor/github.com/containers/storage/drivers/fsdiff.go b/vendor/github.com/containers/storage/drivers/fsdiff.go index 74e43257..e94947d7 100644 --- a/vendor/github.com/containers/storage/drivers/fsdiff.go +++ b/vendor/github.com/containers/storage/drivers/fsdiff.go @@ -36,25 +36,25 @@ type NaiveDiffDriver struct { // ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) // DiffSize(id, parent string) (size int64, err error) func NewNaiveDiffDriver(driver ProtoDriver, uidMaps, gidMaps []idtools.IDMap) Driver { - gdw := &NaiveDiffDriver{ - ProtoDriver: driver, - uidMaps: uidMaps, - gidMaps: gidMaps, - } - return gdw + return &NaiveDiffDriver{ProtoDriver: driver, + uidMaps: uidMaps, + gidMaps: gidMaps} } // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err error) { - layerFs, err := gdw.Get(id, "") + startTime := time.Now() + driver := gdw.ProtoDriver + + layerFs, err := driver.Get(id, "") if err != nil { return nil, err } defer func() { if err != nil { - gdw.Put(id) + driver.Put(id) } }() @@ -65,16 +65,16 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err err } return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() - gdw.Put(id) + driver.Put(id) return err }), nil } - parentFs, err := gdw.Get(parent, "") + parentFs, err := driver.Get(parent, "") if err != nil { return nil, err } - defer gdw.Put(parent) + defer driver.Put(parent) changes, err := archive.ChangesDirs(layerFs, parentFs) if err != nil { @@ -88,7 +88,13 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err err return ioutils.NewReadCloserWrapper(archive, func() error { err := archive.Close() - gdw.Put(id) + driver.Put(id) + + // NaiveDiffDriver compares file metadata with parent layers. Parent layers + // are extracted from tar's with full second precision on modified time. + // We need this hack here to make sure calls within same second receive + // correct result. + time.Sleep(time.Until(startTime.Truncate(time.Second).Add(time.Second))) return err }), nil } @@ -96,20 +102,22 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err err // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. func (gdw *NaiveDiffDriver) Changes(id, parent string) ([]archive.Change, error) { - layerFs, err := gdw.Get(id, "") + driver := gdw.ProtoDriver + + layerFs, err := driver.Get(id, "") if err != nil { return nil, err } - defer gdw.Put(id) + defer driver.Put(id) parentFs := "" if parent != "" { - parentFs, err = gdw.Get(parent, "") + parentFs, err = driver.Get(parent, "") if err != nil { return nil, err } - defer gdw.Put(parent) + defer driver.Put(parent) } return archive.ChangesDirs(layerFs, parentFs) @@ -119,12 +127,14 @@ func (gdw *NaiveDiffDriver) Changes(id, parent string) ([]archive.Change, error) // layer with the specified id and parent, returning the size of the // new layer in bytes. func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff io.Reader) (size int64, err error) { + driver := gdw.ProtoDriver + // Mount the root filesystem so we can apply the diff/layer. - layerFs, err := gdw.Get(id, "") + layerFs, err := driver.Get(id, "") if err != nil { return } - defer gdw.Put(id) + defer driver.Put(id) options := &archive.TarOptions{UIDMaps: gdw.uidMaps, GIDMaps: gdw.gidMaps} @@ -142,16 +152,18 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff io.Reader) (size i // and its parent and returns the size in bytes of the changes // relative to its base filesystem directory. func (gdw *NaiveDiffDriver) DiffSize(id, parent string) (size int64, err error) { + driver := gdw.ProtoDriver + changes, err := gdw.Changes(id, parent) if err != nil { return } - layerFs, err := gdw.Get(id, "") + layerFs, err := driver.Get(id, "") if err != nil { return } - defer gdw.Put(id) + defer driver.Put(id) return archive.ChangesSize(layerFs, changes), nil } diff --git a/vendor/github.com/containers/storage/drivers/overlay/check.go b/vendor/github.com/containers/storage/drivers/overlay/check.go new file mode 100644 index 00000000..2f6ea14f --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/overlay/check.go @@ -0,0 +1,79 @@ +// +build linux + +package overlay + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + + "github.com/containers/storage/pkg/system" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +// hasOpaqueCopyUpBug checks whether the filesystem has a bug +// which copies up the opaque flag when copying up an opaque +// directory. When this bug exists naive diff should be used. +func hasOpaqueCopyUpBug(d string) error { + td, err := ioutil.TempDir(d, "opaque-bug-check") + if err != nil { + return err + } + defer func() { + if err := os.RemoveAll(td); err != nil { + logrus.Warnf("Failed to remove check directory %v: %v", td, err) + } + }() + + // Make directories l1/d, l2/d, l3, work, merged + if err := os.MkdirAll(filepath.Join(td, "l1", "d"), 0755); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(td, "l2", "d"), 0755); err != nil { + return err + } + if err := os.Mkdir(filepath.Join(td, "l3"), 0755); err != nil { + return err + } + if err := os.Mkdir(filepath.Join(td, "work"), 0755); err != nil { + return err + } + if err := os.Mkdir(filepath.Join(td, "merged"), 0755); err != nil { + return err + } + + // Mark l2/d as opaque + if err := system.Lsetxattr(filepath.Join(td, "l2", "d"), "trusted.overlay.opaque", []byte("y"), 0); err != nil { + return errors.Wrap(err, "failed to set opaque flag on middle layer") + } + + opts := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", path.Join(td, "l2"), path.Join(td, "l1"), path.Join(td, "l3"), path.Join(td, "work")) + if err := unix.Mount("overlay", filepath.Join(td, "merged"), "overlay", 0, opts); err != nil { + return errors.Wrap(err, "failed to mount overlay") + } + defer func() { + if err := unix.Unmount(filepath.Join(td, "merged"), 0); err != nil { + logrus.Warnf("Failed to unmount check directory %v: %v", filepath.Join(td, "merged"), err) + } + }() + + // Touch file in d to force copy up of opaque directory "d" from "l2" to "l3" + if err := ioutil.WriteFile(filepath.Join(td, "merged", "d", "f"), []byte{}, 0644); err != nil { + return errors.Wrap(err, "failed to write to merged directory") + } + + // Check l3/d does not have opaque flag + xattrOpaque, err := system.Lgetxattr(filepath.Join(td, "l3", "d"), "trusted.overlay.opaque") + if err != nil { + return errors.Wrap(err, "failed to read opaque flag on upper layer") + } + if string(xattrOpaque) == "y" { + return errors.New("opaque flag erroneously copied up, consider update to kernel 4.8 or later to fix") + } + + return nil +} diff --git a/vendor/github.com/containers/storage/drivers/overlay/mount.go b/vendor/github.com/containers/storage/drivers/overlay/mount.go index 1b53f0c8..7e20c348 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/mount.go +++ b/vendor/github.com/containers/storage/drivers/overlay/mount.go @@ -9,9 +9,9 @@ import ( "fmt" "os" "runtime" - "syscall" "github.com/containers/storage/pkg/reexec" + "golang.org/x/sys/unix" ) func init() { @@ -31,12 +31,12 @@ type mountOptions struct { Flag uint32 } -func mountFrom(dir, device, target, mType, label string) error { +func mountFrom(dir, device, target, mType string, flags uintptr, label string) error { options := &mountOptions{ Device: device, Target: target, Type: mType, - Flag: 0, + Flag: uint32(flags), Label: label, } @@ -51,16 +51,18 @@ func mountFrom(dir, device, target, mType, label string) error { cmd.Stderr = output if err := cmd.Start(); err != nil { + w.Close() return fmt.Errorf("mountfrom error on re-exec cmd: %v", err) } //write the options to the pipe for the untar exec to read if err := json.NewEncoder(w).Encode(options); err != nil { + w.Close() return fmt.Errorf("mountfrom json encode to pipe failed: %v", err) } w.Close() if err := cmd.Wait(); err != nil { - return fmt.Errorf("mountfrom re-exec error: %v: output: %s", err, output) + return fmt.Errorf("mountfrom re-exec error: %v: output: %v", err, output) } return nil } @@ -80,7 +82,7 @@ func mountFromMain() { fatal(err) } - if err := syscall.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil { + if err := unix.Mount(options.Device, options.Target, options.Type, uintptr(options.Flag), options.Label); err != nil { fatal(err) } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index f5e302d7..29ec7b73 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -13,21 +13,26 @@ import ( "path/filepath" "strconv" "strings" - "syscall" - - "github.com/sirupsen/logrus" + "sync" "github.com/containers/storage/drivers" + "github.com/containers/storage/drivers/overlayutils" + "github.com/containers/storage/drivers/quota" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/directory" + "github.com/containers/storage/pkg/fsutils" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/locker" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" "github.com/containers/storage/pkg/parsers/kernel" - + "github.com/containers/storage/pkg/system" + units "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) var ( @@ -43,7 +48,7 @@ var ( // Each container/image has at least a "diff" directory and "link" file. // If there is also a "lower" file when there are diff layers -// below as well as "merged" and "work" directories. The "diff" directory +// below as well as "merged" and "work" directories. The "diff" directory // has the upper layer of the overlay and is used to capture any // changes to the layer. The "lower" file contains all the lower layer // mounts separated by ":" and ordered from uppermost to lowermost @@ -77,27 +82,44 @@ const ( idLength = 26 ) +type overlayOptions struct { + overrideKernelCheck bool + imageStores []string + quota quota.Quota +} + // Driver contains information about the home directory and the list of active mounts that are created using this driver. type Driver struct { - name string - home string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap - ctr *graphdriver.RefCounter - opts *overlayOptions + name string + home string + uidMaps []idtools.IDMap + gidMaps []idtools.IDMap + ctr *graphdriver.RefCounter + quotaCtl *quota.Control + options overlayOptions + naiveDiff graphdriver.DiffDriver + supportsDType bool + locker *locker.Locker } -var backingFs = "" +var ( + backingFs = "" + projectQuotaSupported = false + + useNaiveDiffLock sync.Once + useNaiveDiffOnly bool +) func init() { - graphdriver.Register("overlay", InitAsOverlay) - graphdriver.Register("overlay2", InitAsOverlay2) + graphdriver.Register("overlay", Init) + graphdriver.Register("overlay2", Init) } -// InitWithName returns the a naive diff driver for the overlay filesystem, -// which returns the passed-in name when asked which driver it is. -func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { - opts, err := parseOptions(name, options) +// Init returns the a native diff driver for overlay filesystem. +// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error. +// If an overlay filesystem is not supported over an existing filesystem then error graphdriver.ErrIncompatibleFS is returned. +func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { + opts, err := parseOptions(options) if err != nil { return nil, err } @@ -115,7 +137,7 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool if !opts.overrideKernelCheck { return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay") } - logrus.Warnf("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update") + logrus.Warn("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update") } fsMagic, err := graphdriver.GetFSMagic(home) @@ -128,9 +150,19 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool // check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs switch fsMagic { - case graphdriver.FsMagicBtrfs, graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs: + case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs: logrus.Errorf("'overlay' is not supported over %s", backingFs) return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s", backingFs) + case graphdriver.FsMagicBtrfs: + // Support for OverlayFS on BTRFS was added in kernel 4.7 + // See https://btrfs.wiki.kernel.org/index.php/Changelog + if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 7, Minor: 0}) < 0 { + if !opts.overrideKernelCheck { + logrus.Errorf("'overlay' requires kernel 4.7 to use on %s", backingFs) + return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' requires kernel 4.7 to use on %s", backingFs) + } + logrus.Warn("Using pre-4.7.0 kernel for overlay on btrfs, may require kernel update") + } } rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) @@ -146,38 +178,47 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool return nil, err } - d := &Driver{ - name: name, - home: home, - uidMaps: uidMaps, - gidMaps: gidMaps, - ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)), - opts: opts, + supportsDType, err := fsutils.SupportsDType(home) + if err != nil { + return nil, err } + if !supportsDType { + logrus.Warn(overlayutils.ErrDTypeNotSupported("overlay", backingFs)) + // TODO: Will make fatal when CRI-O Has AMI built on RHEL7.4 + // return nil, overlayutils.ErrDTypeNotSupported("overlay", backingFs) + } + + d := &Driver{ + name: "overlay", + home: home, + uidMaps: uidMaps, + gidMaps: gidMaps, + ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)), + supportsDType: supportsDType, + locker: locker.New(), + options: *opts, + } + + d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps) + + if backingFs == "xfs" { + // Try to enable project quota support over xfs. + if d.quotaCtl, err = quota.NewControl(home); err == nil { + projectQuotaSupported = true + } else if opts.quota.Size > 0 { + return nil, fmt.Errorf("Storage option overlay.size not supported. Filesystem does not support Project Quota: %v", err) + } + } else if opts.quota.Size > 0 { + // if xfs is not the backing fs then error out if the storage-opt overlay.size is used. + return nil, fmt.Errorf("Storage Option overlay.size only supported for backingFS XFS. Found %v", backingFs) + } + + logrus.Debugf("backingFs=%s, projectQuotaSupported=%v", backingFs, projectQuotaSupported) return d, nil } -// InitAsOverlay returns the a naive diff driver for overlay filesystem. -// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error. -// If a overlay filesystem is not supported over a existing filesystem then error graphdriver.ErrIncompatibleFS is returned. -func InitAsOverlay(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { - return InitWithName("overlay", home, options, uidMaps, gidMaps) -} - -// InitAsOverlay2 returns the a naive diff driver for overlay filesystem. -// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error. -// If a overlay filesystem is not supported over a existing filesystem then error graphdriver.ErrIncompatibleFS is returned. -func InitAsOverlay2(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { - return InitWithName("overlay2", home, options, uidMaps, gidMaps) -} - -type overlayOptions struct { - overrideKernelCheck bool - imageStores []string -} - -func parseOptions(name string, options []string) (*overlayOptions, error) { +func parseOptions(options []string) (*overlayOptions, error) { o := &overlayOptions{} for _, option := range options { key, val, err := parsers.ParseKeyValueOpt(option) @@ -187,28 +228,37 @@ func parseOptions(name string, options []string) (*overlayOptions, error) { key = strings.ToLower(key) switch key { case "overlay.override_kernel_check", "overlay2.override_kernel_check": + logrus.Debugf("overlay: overide_kernelcheck=%s", val) o.overrideKernelCheck, err = strconv.ParseBool(val) if err != nil { return nil, err } + case "overlay.size", "overlay2.size": + logrus.Debugf("overlay: size=%s", val) + size, err := units.RAMInBytes(val) + if err != nil { + return nil, err + } + o.quota.Size = uint64(size) case "overlay.imagestore", "overlay2.imagestore": + logrus.Debugf("overlay: imagestore=%s", val) // Additional read only image stores to use for lower paths for _, store := range strings.Split(val, ",") { store = filepath.Clean(store) if !filepath.IsAbs(store) { - return nil, fmt.Errorf("%s: image path %q is not absolute. Can not be relative", name, store) + return nil, fmt.Errorf("overlay: image path %q is not absolute. Can not be relative", store) } st, err := os.Stat(store) if err != nil { - return nil, fmt.Errorf("%s: Can't stat imageStore dir %s: %v", name, store, err) + return nil, fmt.Errorf("overlay: can't stat imageStore dir %s: %v", store, err) } if !st.IsDir() { - return nil, fmt.Errorf("%s: image path %q must be a directory", name, store) + return nil, fmt.Errorf("overlay: image path %q must be a directory", store) } o.imageStores = append(o.imageStores, store) } default: - return nil, fmt.Errorf("%s: Unknown option %s", name, key) + return nil, fmt.Errorf("overlay: Unknown option %s", key) } } return o, nil @@ -235,6 +285,16 @@ func supportsOverlay() error { return errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.") } +func useNaiveDiff(home string) bool { + useNaiveDiffLock.Do(func() { + if err := hasOpaqueCopyUpBug(home); err != nil { + logrus.Warnf("Not using native diff for overlay: %v", err) + useNaiveDiffOnly = true + } + }) + return useNaiveDiffOnly +} + func (d *Driver) String() string { return d.name } @@ -244,6 +304,8 @@ func (d *Driver) String() string { func (d *Driver) Status() [][2]string { return [][2]string{ {"Backing Filesystem", backingFs}, + {"Supports d_type", strconv.FormatBool(d.supportsDType)}, + {"Native Overlay Diff", strconv.FormatBool(!useNaiveDiff(d.home))}, } } @@ -281,18 +343,39 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + if opts != nil && len(opts.StorageOpt) != 0 && !projectQuotaSupported { + return fmt.Errorf("--storage-opt is supported only for overlay over xfs with 'pquota' mount option") + } + + if opts == nil { + opts = &graphdriver.CreateOpts{ + StorageOpt: map[string]string{}, + } + } + + if _, ok := opts.StorageOpt["size"]; !ok { + if opts.StorageOpt == nil { + opts.StorageOpt = map[string]string{} + } + opts.StorageOpt["size"] = strconv.FormatUint(d.options.quota.Size, 10) + } + + return d.create(id, parent, opts) } // Create is used to create the upper, lower, and merge directories required for overlay fs for a given id. // The parent filesystem is used to configure these directories for the overlay. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) (retErr error) { - - if len(storageOpt) != 0 { - return fmt.Errorf("--storage-opt is not supported for overlay") +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { + if opts != nil && len(opts.StorageOpt) != 0 { + if _, ok := opts.StorageOpt["size"]; ok { + return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers") + } } + return d.create(id, parent, opts) +} +func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr error) { dir := d.dir(id) rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) @@ -313,6 +396,20 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str } }() + if opts != nil && len(opts.StorageOpt) > 0 { + driver := &Driver{} + if err := d.parseStorageOpt(opts.StorageOpt, driver); err != nil { + return err + } + + if driver.options.quota.Size > 0 { + // Set container disk quota limit + if err := d.quotaCtl.SetQuota(dir, driver.options.quota); err != nil { + return err + } + } + } + if err := idtools.MkdirAs(path.Join(dir, "diff"), 0755, rootUID, rootGID); err != nil { return err } @@ -352,6 +449,26 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str return nil } +// Parse overlay storage options +func (d *Driver) parseStorageOpt(storageOpt map[string]string, driver *Driver) error { + // Read size to set the disk project quota per container + for key, val := range storageOpt { + key := strings.ToLower(key) + switch key { + case "size": + size, err := units.RAMInBytes(val) + if err != nil { + return err + } + driver.options.quota.Size = uint64(size) + default: + return fmt.Errorf("Unknown option %s", key) + } + } + + return nil +} + func (d *Driver) getLower(parent string) (string, error) { parentDir := d.dir(parent) @@ -378,11 +495,11 @@ func (d *Driver) getLower(parent string) (string, error) { return strings.Join(lowers, ":"), nil } -func (d *Driver) dir(val string) string { - newpath := path.Join(d.home, val) +func (d *Driver) dir(id string) string { + newpath := path.Join(d.home, id) if _, err := os.Stat(newpath); err != nil { for _, p := range d.AdditionalImageStores() { - l := path.Join(p, d.name, val) + l := path.Join(p, d.name, id) _, err = os.Stat(l) if err == nil { return l @@ -412,6 +529,8 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) { // Remove cleans the directories that are created for this id. func (d *Driver) Remove(id string) error { + d.locker.Lock(id) + defer d.locker.Unlock(id) dir := d.dir(id) lid, err := ioutil.ReadFile(path.Join(dir, "link")) if err == nil { @@ -420,14 +539,16 @@ func (d *Driver) Remove(id string) error { } } - if err := os.RemoveAll(dir); err != nil && !os.IsNotExist(err) { + if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) { return err } return nil } // Get creates and mounts the required file system for the given id and returns the mount path. -func (d *Driver) Get(id string, mountLabel string) (s string, err error) { +func (d *Driver) Get(id, mountLabel string) (_ string, retErr error) { + d.locker.Lock(id) + defer d.locker.Unlock(id) dir := d.dir(id) if _, err := os.Stat(dir); err != nil { return "", err @@ -459,7 +580,7 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) { return "", fmt.Errorf("Can't stat lower layer %q: %v", newpath, err) } } else { - lower = l + lower = newpath } if newlowers == "" { newlowers = lower @@ -473,22 +594,42 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) { return mergedDir, nil } defer func() { - if err != nil { + if retErr != nil { if c := d.ctr.Decrement(mergedDir); c <= 0 { - syscall.Unmount(mergedDir, 0) + if mntErr := unix.Unmount(mergedDir, 0); mntErr != nil { + logrus.Errorf("error unmounting %v: %v", mergedDir, mntErr) + } } } }() workDir := path.Join(dir, "work") - opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", newlowers, path.Join(id, "diff"), path.Join(id, "work")) - mountLabel = label.FormatMountLabel(opts, mountLabel) - if len(mountLabel) > syscall.Getpagesize() { - return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountLabel)) - } + opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", newlowers, diffDir, workDir) + mountData := label.FormatMountLabel(opts, mountLabel) + mount := unix.Mount + mountTarget := mergedDir - if err := mountFrom(d.home, "overlay", path.Join(id, "merged"), "overlay", mountLabel); err != nil { - return "", fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err) + pageSize := unix.Getpagesize() + + // Use relative paths and mountFrom when the mount data has exceeded + // the page size. The mount syscall fails if the mount data cannot + // fit within a page and relative links make the mount data much + // smaller at the expense of requiring a fork exec to chroot. + if len(mountData) > pageSize { + //FIXME: We need to figure out to get this to work with additional stores + opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", string(lowers), path.Join(id, "diff"), path.Join(id, "work")) + mountData = label.FormatMountLabel(opts, mountLabel) + if len(mountData) > pageSize { + return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData)) + } + + mount = func(source string, target string, mType string, flags uintptr, label string) error { + return mountFrom(d.home, source, target, mType, flags, label) + } + mountTarget = path.Join(id, "merged") + } + if err := mount("overlay", mountTarget, "overlay", 0, mountData); err != nil { + return "", fmt.Errorf("error creating overlay mount to %s: %v", mountTarget, err) } // chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a @@ -507,19 +648,17 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) { // Put unmounts the mount path created for the give id. func (d *Driver) Put(id string) error { + d.locker.Lock(id) + defer d.locker.Unlock(id) mountpoint := path.Join(d.dir(id), "merged") if count := d.ctr.Decrement(mountpoint); count > 0 { return nil } - err := syscall.Unmount(mountpoint, 0) + err := unix.Unmount(mountpoint, unix.MNT_DETACH) if err != nil { - if _, err := ioutil.ReadFile(path.Join(d.dir(id), lowerFile)); err != nil { - // We didn't have a "lower" directory, so we weren't mounting a "merged" directory anyway - return nil - } - logrus.Debugf("Failed to unmount %s %s: %v", id, d.name, err) + logrus.Debugf("Failed to unmount %s overlay: %s - %v", id, mountpoint, err) } - return err + return nil } // Exists checks to see if the id is already mounted. @@ -528,8 +667,33 @@ func (d *Driver) Exists(id string) bool { return err == nil } +// isParent returns if the passed in parent is the direct parent of the passed in layer +func (d *Driver) isParent(id, parent string) bool { + lowers, err := d.getLowerDirs(id) + if err != nil { + return false + } + if parent == "" && len(lowers) > 0 { + return false + } + + parentDir := d.dir(parent) + var ld string + if len(lowers) > 0 { + ld = filepath.Dir(lowers[0]) + } + if ld == "" && parent == "" { + return true + } + return ld == parentDir +} + // ApplyDiff applies the new layer into a root func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64, err error) { + if !d.isParent(id, parent) { + return d.naiveDiff.ApplyDiff(id, parent, diff) + } + applyDir := d.getDiffPath(id) logrus.Debugf("Applying tar in %s", applyDir) @@ -542,7 +706,7 @@ func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64 return 0, err } - return d.DiffSize(id, parent) + return directory.Size(applyDir) } func (d *Driver) getDiffPath(id string) string { @@ -555,12 +719,19 @@ func (d *Driver) getDiffPath(id string) string { // and its parent and returns the size in bytes of the changes // relative to its base filesystem directory. func (d *Driver) DiffSize(id, parent string) (size int64, err error) { + if useNaiveDiff(d.home) || !d.isParent(id, parent) { + return d.naiveDiff.DiffSize(id, parent) + } return directory.Size(d.getDiffPath(id)) } // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) { + if useNaiveDiff(d.home) || !d.isParent(id, parent) { + return d.naiveDiff.Diff(id, parent) + } + diffPath := d.getDiffPath(id) logrus.Debugf("Tar with options on %s", diffPath) return archive.TarWithOptions(diffPath, &archive.TarOptions{ @@ -574,6 +745,9 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) { // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { + if useNaiveDiff(d.home) || !d.isParent(id, parent) { + return d.naiveDiff.Changes(id, parent) + } // Overlay doesn't have snapshots, so we need to get changes from all parent // layers. diffPath := d.getDiffPath(id) @@ -587,5 +761,5 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { - return d.opts.imageStores + return d.options.imageStores } diff --git a/vendor/github.com/containers/storage/drivers/overlay/randomid.go b/vendor/github.com/containers/storage/drivers/overlay/randomid.go index 975b3a50..fc565ef0 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/randomid.go +++ b/vendor/github.com/containers/storage/drivers/overlay/randomid.go @@ -12,6 +12,7 @@ import ( "time" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // generateID creates a new random string identifier with the given length @@ -69,7 +70,7 @@ func retryOnError(err error) bool { case *os.PathError: return retryOnError(err.Err) // unpack the target error case syscall.Errno: - if err == syscall.EPERM { + if err == unix.EPERM { // EPERM represents an entropy pool exhaustion, a condition under // which we backoff and retry. return true diff --git a/vendor/github.com/containers/storage/drivers/overlayutils/overlayutils.go b/vendor/github.com/containers/storage/drivers/overlayutils/overlayutils.go new file mode 100644 index 00000000..46773364 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/overlayutils/overlayutils.go @@ -0,0 +1,18 @@ +// +build linux + +package overlayutils + +import ( + "errors" + "fmt" +) + +// ErrDTypeNotSupported denotes that the backing filesystem doesn't support d_type. +func ErrDTypeNotSupported(driver, backingFs string) error { + msg := fmt.Sprintf("%s: the backing %s filesystem is formatted without d_type support, which leads to incorrect behavior.", driver, backingFs) + if backingFs == "xfs" { + msg += " Reformat the filesystem with ftype=1 to enable d_type support." + } + msg += " Running without d_type is not supported." + return errors.New(msg) +} diff --git a/vendor/github.com/containers/storage/drivers/plugin.go b/vendor/github.com/containers/storage/drivers/plugin.go deleted file mode 100644 index a76aae6e..00000000 --- a/vendor/github.com/containers/storage/drivers/plugin.go +++ /dev/null @@ -1,32 +0,0 @@ -// +build experimental - -package graphdriver - -import ( - "fmt" - "io" - - "github.com/containers/storage/pkg/plugins" -) - -type pluginClient interface { - // Call calls the specified method with the specified arguments for the plugin. - Call(string, interface{}, interface{}) error - // Stream calls the specified method with the specified arguments for the plugin and returns the response IO stream - Stream(string, interface{}) (io.ReadCloser, error) - // SendFile calls the specified method, and passes through the IO stream - SendFile(string, io.Reader, interface{}) error -} - -func lookupPlugin(name, home string, opts []string) (Driver, error) { - pl, err := plugins.Get(name, "GraphDriver") - if err != nil { - return nil, fmt.Errorf("Error looking up graphdriver plugin %s: %v", name, err) - } - return newPluginDriver(name, home, opts, pl.Client()) -} - -func newPluginDriver(name, home string, opts []string, c pluginClient) (Driver, error) { - proxy := &graphDriverProxy{name, c} - return proxy, proxy.Init(home, opts) -} diff --git a/vendor/github.com/containers/storage/drivers/plugin_unsupported.go b/vendor/github.com/containers/storage/drivers/plugin_unsupported.go deleted file mode 100644 index daa7a170..00000000 --- a/vendor/github.com/containers/storage/drivers/plugin_unsupported.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build !experimental - -package graphdriver - -func lookupPlugin(name, home string, opts []string) (Driver, error) { - return nil, ErrNotSupported -} diff --git a/vendor/github.com/containers/storage/drivers/proxy.go b/vendor/github.com/containers/storage/drivers/proxy.go deleted file mode 100644 index e4f742fb..00000000 --- a/vendor/github.com/containers/storage/drivers/proxy.go +++ /dev/null @@ -1,226 +0,0 @@ -// +build experimental - -package graphdriver - -import ( - "fmt" - "io" - - "github.com/containers/storage/pkg/archive" - "github.com/pkg/errors" -) - -type graphDriverProxy struct { - name string - client pluginClient -} - -type graphDriverRequest struct { - ID string `json:",omitempty"` - Parent string `json:",omitempty"` - MountLabel string `json:",omitempty"` -} - -type graphDriverResponse struct { - Err string `json:",omitempty"` - Dir string `json:",omitempty"` - Exists bool `json:",omitempty"` - Status [][2]string `json:",omitempty"` - Changes []archive.Change `json:",omitempty"` - Size int64 `json:",omitempty"` - Metadata map[string]string `json:",omitempty"` -} - -type graphDriverInitRequest struct { - Home string - Opts []string -} - -func (d *graphDriverProxy) Init(home string, opts []string) error { - args := &graphDriverInitRequest{ - Home: home, - Opts: opts, - } - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Init", args, &ret); err != nil { - return err - } - if ret.Err != "" { - return errors.New(ret.Err) - } - return nil -} - -func (d *graphDriverProxy) String() string { - return d.name -} - -func (d *graphDriverProxy) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - args := &graphDriverRequest{ - ID: id, - Parent: parent, - MountLabel: mountLabel, - } - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.CreateReadWrite", args, &ret); err != nil { - return err - } - if ret.Err != "" { - return errors.New(ret.Err) - } - return nil -} - -func (d *graphDriverProxy) Create(id, parent, mountLabel string, storageOpt map[string]string) error { - args := &graphDriverRequest{ - ID: id, - Parent: parent, - MountLabel: mountLabel, - } - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Create", args, &ret); err != nil { - return err - } - if ret.Err != "" { - return errors.New(ret.Err) - } - return nil -} - -func (d *graphDriverProxy) Remove(id string) error { - args := &graphDriverRequest{ID: id} - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Remove", args, &ret); err != nil { - return err - } - if ret.Err != "" { - return errors.New(ret.Err) - } - return nil -} - -func (d *graphDriverProxy) Get(id, mountLabel string) (string, error) { - args := &graphDriverRequest{ - ID: id, - MountLabel: mountLabel, - } - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Get", args, &ret); err != nil { - return "", err - } - var err error - if ret.Err != "" { - err = errors.New(ret.Err) - } - return ret.Dir, err -} - -func (d *graphDriverProxy) Put(id string) error { - args := &graphDriverRequest{ID: id} - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Put", args, &ret); err != nil { - return err - } - if ret.Err != "" { - return errors.New(ret.Err) - } - return nil -} - -func (d *graphDriverProxy) Exists(id string) bool { - args := &graphDriverRequest{ID: id} - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Exists", args, &ret); err != nil { - return false - } - return ret.Exists -} - -func (d *graphDriverProxy) Status() [][2]string { - args := &graphDriverRequest{} - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Status", args, &ret); err != nil { - return nil - } - return ret.Status -} - -func (d *graphDriverProxy) Metadata(id string) (map[string]string, error) { - args := &graphDriverRequest{ - ID: id, - } - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Metadata", args, &ret); err != nil { - return nil, err - } - if ret.Err != "" { - return nil, errors.New(ret.Err) - } - return ret.Metadata, nil -} - -func (d *graphDriverProxy) Cleanup() error { - args := &graphDriverRequest{} - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Cleanup", args, &ret); err != nil { - return nil - } - if ret.Err != "" { - return errors.New(ret.Err) - } - return nil -} - -func (d *graphDriverProxy) Diff(id, parent string) (io.ReadCloser, error) { - args := &graphDriverRequest{ - ID: id, - Parent: parent, - } - body, err := d.client.Stream("GraphDriver.Diff", args) - if err != nil { - return nil, err - } - return io.ReadClose(body), nil -} - -func (d *graphDriverProxy) Changes(id, parent string) ([]archive.Change, error) { - args := &graphDriverRequest{ - ID: id, - Parent: parent, - } - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.Changes", args, &ret); err != nil { - return nil, err - } - if ret.Err != "" { - return nil, errors.New(ret.Err) - } - - return ret.Changes, nil -} - -func (d *graphDriverProxy) ApplyDiff(id, parent string, diff io.Reader) (int64, error) { - var ret graphDriverResponse - if err := d.client.SendFile(fmt.Sprintf("GraphDriver.ApplyDiff?id=%s&parent=%s", id, parent), diff, &ret); err != nil { - return -1, err - } - if ret.Err != "" { - return -1, errors.New(ret.Err) - } - return ret.Size, nil -} - -func (d *graphDriverProxy) DiffSize(id, parent string) (int64, error) { - args := &graphDriverRequest{ - ID: id, - Parent: parent, - } - var ret graphDriverResponse - if err := d.client.Call("GraphDriver.DiffSize", args, &ret); err != nil { - return -1, err - } - if ret.Err != "" { - return -1, errors.New(ret.Err) - } - return ret.Size, nil -} diff --git a/vendor/github.com/containers/storage/drivers/quota/projectquota.go b/vendor/github.com/containers/storage/drivers/quota/projectquota.go new file mode 100644 index 00000000..93e74437 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/quota/projectquota.go @@ -0,0 +1,337 @@ +// +build linux + +// +// projectquota.go - implements XFS project quota controls +// for setting quota limits on a newly created directory. +// It currently supports the legacy XFS specific ioctls. +// +// TODO: use generic quota control ioctl FS_IOC_FS{GET,SET}XATTR +// for both xfs/ext4 for kernel version >= v4.5 +// + +package quota + +/* +#include +#include +#include +#include +#include + +#ifndef FS_XFLAG_PROJINHERIT +struct fsxattr { + __u32 fsx_xflags; + __u32 fsx_extsize; + __u32 fsx_nextents; + __u32 fsx_projid; + unsigned char fsx_pad[12]; +}; +#define FS_XFLAG_PROJINHERIT 0x00000200 +#endif +#ifndef FS_IOC_FSGETXATTR +#define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr) +#endif +#ifndef FS_IOC_FSSETXATTR +#define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr) +#endif + +#ifndef PRJQUOTA +#define PRJQUOTA 2 +#endif +#ifndef XFS_PROJ_QUOTA +#define XFS_PROJ_QUOTA 2 +#endif +#ifndef Q_XSETPQLIM +#define Q_XSETPQLIM QCMD(Q_XSETQLIM, PRJQUOTA) +#endif +#ifndef Q_XGETPQUOTA +#define Q_XGETPQUOTA QCMD(Q_XGETQUOTA, PRJQUOTA) +#endif +*/ +import "C" +import ( + "fmt" + "io/ioutil" + "path" + "path/filepath" + "unsafe" + + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +// Quota limit params - currently we only control blocks hard limit +type Quota struct { + Size uint64 +} + +// Control - Context to be used by storage driver (e.g. overlay) +// who wants to apply project quotas to container dirs +type Control struct { + backingFsBlockDev string + nextProjectID uint32 + quotas map[string]uint32 +} + +// NewControl - initialize project quota support. +// Test to make sure that quota can be set on a test dir and find +// the first project id to be used for the next container create. +// +// Returns nil (and error) if project quota is not supported. +// +// First get the project id of the home directory. +// This test will fail if the backing fs is not xfs. +// +// xfs_quota tool can be used to assign a project id to the driver home directory, e.g.: +// echo 999:/var/lib/containers/storage/overlay >> /etc/projects +// echo storage:999 >> /etc/projid +// xfs_quota -x -c 'project -s storage' / +// +// In that case, the home directory project id will be used as a "start offset" +// and all containers will be assigned larger project ids (e.g. >= 1000). +// This is a way to prevent xfs_quota management from conflicting with containers/storage. +// +// Then try to create a test directory with the next project id and set a quota +// on it. If that works, continue to scan existing containers to map allocated +// project ids. +// +func NewControl(basePath string) (*Control, error) { + // + // Get project id of parent dir as minimal id to be used by driver + // + minProjectID, err := getProjectID(basePath) + if err != nil { + return nil, err + } + minProjectID++ + + // + // create backing filesystem device node + // + backingFsBlockDev, err := makeBackingFsDev(basePath) + if err != nil { + return nil, err + } + + // + // Test if filesystem supports project quotas by trying to set + // a quota on the first available project id + // + quota := Quota{ + Size: 0, + } + if err := setProjectQuota(backingFsBlockDev, minProjectID, quota); err != nil { + return nil, err + } + + q := Control{ + backingFsBlockDev: backingFsBlockDev, + nextProjectID: minProjectID + 1, + quotas: make(map[string]uint32), + } + + // + // get first project id to be used for next container + // + err = q.findNextProjectID(basePath) + if err != nil { + return nil, err + } + + logrus.Debugf("NewControl(%s): nextProjectID = %d", basePath, q.nextProjectID) + return &q, nil +} + +// SetQuota - assign a unique project id to directory and set the quota limits +// for that project id +func (q *Control) SetQuota(targetPath string, quota Quota) error { + + projectID, ok := q.quotas[targetPath] + if !ok { + projectID = q.nextProjectID + + // + // assign project id to new container directory + // + err := setProjectID(targetPath, projectID) + if err != nil { + return err + } + + q.quotas[targetPath] = projectID + q.nextProjectID++ + } + + // + // set the quota limit for the container's project id + // + logrus.Debugf("SetQuota(%s, %d): projectID=%d", targetPath, quota.Size, projectID) + return setProjectQuota(q.backingFsBlockDev, projectID, quota) +} + +// setProjectQuota - set the quota for project id on xfs block device +func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) error { + var d C.fs_disk_quota_t + d.d_version = C.FS_DQUOT_VERSION + d.d_id = C.__u32(projectID) + d.d_flags = C.XFS_PROJ_QUOTA + + d.d_fieldmask = C.FS_DQ_BHARD | C.FS_DQ_BSOFT + d.d_blk_hardlimit = C.__u64(quota.Size / 512) + d.d_blk_softlimit = d.d_blk_hardlimit + + var cs = C.CString(backingFsBlockDev) + defer C.free(unsafe.Pointer(cs)) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XSETPQLIM, + uintptr(unsafe.Pointer(cs)), uintptr(d.d_id), + uintptr(unsafe.Pointer(&d)), 0, 0) + if errno != 0 { + return fmt.Errorf("Failed to set quota limit for projid %d on %s: %v", + projectID, backingFsBlockDev, errno.Error()) + } + + return nil +} + +// GetQuota - get the quota limits of a directory that was configured with SetQuota +func (q *Control) GetQuota(targetPath string, quota *Quota) error { + + projectID, ok := q.quotas[targetPath] + if !ok { + return fmt.Errorf("quota not found for path : %s", targetPath) + } + + // + // get the quota limit for the container's project id + // + var d C.fs_disk_quota_t + + var cs = C.CString(q.backingFsBlockDev) + defer C.free(unsafe.Pointer(cs)) + + _, _, errno := unix.Syscall6(unix.SYS_QUOTACTL, C.Q_XGETPQUOTA, + uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)), + uintptr(unsafe.Pointer(&d)), 0, 0) + if errno != 0 { + return fmt.Errorf("Failed to get quota limit for projid %d on %s: %v", + projectID, q.backingFsBlockDev, errno.Error()) + } + quota.Size = uint64(d.d_blk_hardlimit) * 512 + + return nil +} + +// getProjectID - get the project id of path on xfs +func getProjectID(targetPath string) (uint32, error) { + dir, err := openDir(targetPath) + if err != nil { + return 0, err + } + defer closeDir(dir) + + var fsx C.struct_fsxattr + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return 0, fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) + } + + return uint32(fsx.fsx_projid), nil +} + +// setProjectID - set the project id of path on xfs +func setProjectID(targetPath string, projectID uint32) error { + dir, err := openDir(targetPath) + if err != nil { + return err + } + defer closeDir(dir) + + var fsx C.struct_fsxattr + _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSGETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return fmt.Errorf("Failed to get projid for %s: %v", targetPath, errno.Error()) + } + fsx.fsx_projid = C.__u32(projectID) + fsx.fsx_xflags |= C.FS_XFLAG_PROJINHERIT + _, _, errno = unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.FS_IOC_FSSETXATTR, + uintptr(unsafe.Pointer(&fsx))) + if errno != 0 { + return fmt.Errorf("Failed to set projid for %s: %v", targetPath, errno.Error()) + } + + return nil +} + +// findNextProjectID - find the next project id to be used for containers +// by scanning driver home directory to find used project ids +func (q *Control) findNextProjectID(home string) error { + files, err := ioutil.ReadDir(home) + if err != nil { + return fmt.Errorf("read directory failed : %s", home) + } + for _, file := range files { + if !file.IsDir() { + continue + } + path := filepath.Join(home, file.Name()) + projid, err := getProjectID(path) + if err != nil { + return err + } + if projid > 0 { + q.quotas[path] = projid + } + if q.nextProjectID <= projid { + q.nextProjectID = projid + 1 + } + } + + return nil +} + +func free(p *C.char) { + C.free(unsafe.Pointer(p)) +} + +func openDir(path string) (*C.DIR, error) { + Cpath := C.CString(path) + defer free(Cpath) + + dir := C.opendir(Cpath) + if dir == nil { + return nil, fmt.Errorf("Can't open dir") + } + return dir, nil +} + +func closeDir(dir *C.DIR) { + if dir != nil { + C.closedir(dir) + } +} + +func getDirFd(dir *C.DIR) uintptr { + return uintptr(C.dirfd(dir)) +} + +// Get the backing block device of the driver home directory +// and create a block device node under the home directory +// to be used by quotactl commands +func makeBackingFsDev(home string) (string, error) { + var stat unix.Stat_t + if err := unix.Stat(home, &stat); err != nil { + return "", err + } + + backingFsBlockDev := path.Join(home, "backingFsBlockDev") + // Re-create just in case someone copied the home directory over to a new device + unix.Unlink(backingFsBlockDev) + if err := unix.Mknod(backingFsBlockDev, unix.S_IFBLK|0600, int(stat.Dev)); err != nil { + return "", fmt.Errorf("Failed to mknod %s: %v", backingFsBlockDev, err) + } + + return backingFsBlockDev, nil +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index a5aefd92..d9e24e2e 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -8,7 +8,7 @@ import ( "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/idtools" - + "github.com/containers/storage/pkg/system" "github.com/opencontainers/selinux/go-selinux/label" ) @@ -25,15 +25,11 @@ func init() { // This sets the home directory for the driver and returns NaiveDiffDriver. func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { d := &Driver{ - home: home, - uidMaps: uidMaps, - gidMaps: gidMaps, + home: home, + idMappings: idtools.NewIDMappingsFromMaps(uidMaps, gidMaps), } - rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) - if err != nil { - return nil, err - } - if err := idtools.MkdirAllAs(home, 0700, rootUID, rootGID); err != nil { + rootIDs := d.idMappings.RootPair() + if err := idtools.MkdirAllAndChown(home, 0700, rootIDs); err != nil { return nil, err } return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil @@ -44,9 +40,8 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap // In order to support layering, files are copied from the parent layer into the new layer. There is no copy-on-write support. // Driver must be wrapped in NaiveDiffDriver to be used as a graphdriver.Driver type Driver struct { - home string - uidMaps []idtools.IDMap - gidMaps []idtools.IDMap + home string + idMappings *idtools.IDMappings } func (d *Driver) String() string { @@ -70,29 +65,26 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return d.Create(id, parent, opts) } // Create prepares the filesystem for the VFS driver and copies the directory for the given id under the parent. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { - if len(storageOpt) != 0 { +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + if opts != nil && len(opts.StorageOpt) != 0 { return fmt.Errorf("--storage-opt is not supported for vfs") } dir := d.dir(id) - rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) - if err != nil { + rootIDs := d.idMappings.RootPair() + if err := idtools.MkdirAllAndChown(filepath.Dir(dir), 0700, rootIDs); err != nil { return err } - if err := idtools.MkdirAllAs(filepath.Dir(dir), 0700, rootUID, rootGID); err != nil { + if err := idtools.MkdirAndChown(dir, 0755, rootIDs); err != nil { return err } - if err := idtools.MkdirAs(dir, 0755, rootUID, rootGID); err != nil { - return err - } - opts := []string{"level:s0"} - if _, mountLabel, err := label.InitLabels(opts); err == nil { + labelOpts := []string{"level:s0"} + if _, mountLabel, err := label.InitLabels(labelOpts); err == nil { label.SetFileLabel(dir, mountLabel) } if parent == "" { @@ -102,10 +94,7 @@ func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]str if err != nil { return fmt.Errorf("%s: %s", parent, err) } - if err := CopyWithTar(parentDir, dir); err != nil { - return err - } - return nil + return CopyWithTar(parentDir, dir) } func (d *Driver) dir(id string) string { @@ -114,10 +103,7 @@ func (d *Driver) dir(id string) string { // Remove deletes the content from the directory for a given id. func (d *Driver) Remove(id string) error { - if err := os.RemoveAll(d.dir(id)); err != nil && !os.IsNotExist(err) { - return err - } - return nil + return system.EnsureRemoveAll(d.dir(id)) } // Get returns the directory for the given id. @@ -146,6 +132,5 @@ func (d *Driver) Exists(id string) bool { // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { - var imageStores []string - return imageStores + return nil } diff --git a/vendor/github.com/containers/storage/drivers/windows/windows.go b/vendor/github.com/containers/storage/drivers/windows/windows.go index a502e963..abe2ac43 100644 --- a/vendor/github.com/containers/storage/drivers/windows/windows.go +++ b/vendor/github.com/containers/storage/drivers/windows/windows.go @@ -6,6 +6,7 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -16,6 +17,7 @@ import ( "strings" "sync" "syscall" + "time" "unsafe" "github.com/Microsoft/go-winio" @@ -29,17 +31,37 @@ import ( "github.com/containers/storage/pkg/longpath" "github.com/containers/storage/pkg/reexec" "github.com/containers/storage/pkg/system" + units "github.com/docker/go-units" "github.com/sirupsen/logrus" - "github.com/vbatts/tar-split/tar/storage" + "golang.org/x/sys/windows" ) // filterDriver is an HCSShim driver type for the Windows Filter driver. const filterDriver = 1 +var ( + // mutatedFiles is a list of files that are mutated by the import process + // and must be backed up and restored. + mutatedFiles = map[string]string{ + "UtilityVM/Files/EFI/Microsoft/Boot/BCD": "bcd.bak", + "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG": "bcd.log.bak", + "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG1": "bcd.log1.bak", + "UtilityVM/Files/EFI/Microsoft/Boot/BCD.LOG2": "bcd.log2.bak", + } + noreexec = false +) + // init registers the windows graph drivers to the register. func init() { graphdriver.Register("windowsfilter", InitFilter) - reexec.Register("storage-windows-write-layer", writeLayer) + // DOCKER_WINDOWSFILTER_NOREEXEC allows for inline processing which makes + // debugging issues in the re-exec codepath significantly easier. + if os.Getenv("DOCKER_WINDOWSFILTER_NOREEXEC") != "" { + logrus.Warnf("WindowsGraphDriver is set to not re-exec. This is intended for debugging purposes only.") + noreexec = true + } else { + reexec.Register("docker-windows-write-layer", writeLayerReexec) + } } type checker struct { @@ -60,13 +82,22 @@ type Driver struct { cache map[string]string } -func isTP5OrOlder() bool { - return system.GetOSVersion().Build <= 14300 -} - // InitFilter returns a new Windows storage filter driver. func InitFilter(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) { logrus.Debugf("WindowsGraphDriver InitFilter at %s", home) + + fsType, err := getFileSystemType(string(home[0])) + if err != nil { + return nil, err + } + if strings.ToLower(fsType) == "refs" { + return nil, fmt.Errorf("%s is on an ReFS volume - ReFS volumes are not supported", home) + } + + if err := idtools.MkdirAllAs(home, 0700, 0, 0); err != nil { + return nil, fmt.Errorf("windowsfilter failed to create '%s': %v", home, err) + } + d := &Driver{ info: hcsshim.DriverInfo{ HomeDir: home, @@ -78,6 +109,37 @@ func InitFilter(home string, options []string, uidMaps, gidMaps []idtools.IDMap) return d, nil } +// win32FromHresult is a helper function to get the win32 error code from an HRESULT +func win32FromHresult(hr uintptr) uintptr { + if hr&0x1fff0000 == 0x00070000 { + return hr & 0xffff + } + return hr +} + +// getFileSystemType obtains the type of a file system through GetVolumeInformation +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364993(v=vs.85).aspx +func getFileSystemType(drive string) (fsType string, hr error) { + var ( + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + procGetVolumeInformation = modkernel32.NewProc("GetVolumeInformationW") + buf = make([]uint16, 255) + size = windows.MAX_PATH + 1 + ) + if len(drive) != 1 { + hr = errors.New("getFileSystemType must be called with a drive letter") + return + } + drive += `:\` + n := uintptr(unsafe.Pointer(nil)) + r0, _, _ := syscall.Syscall9(procGetVolumeInformation.Addr(), 8, uintptr(unsafe.Pointer(windows.StringToUTF16Ptr(drive))), n, n, n, n, n, uintptr(unsafe.Pointer(&buf[0])), uintptr(size), 0) + if int32(r0) < 0 { + hr = syscall.Errno(win32FromHresult(r0)) + } + fsType = windows.UTF16ToString(buf) + return +} + // String returns the string representation of a driver. This should match // the name the graph driver has been registered with. func (d *Driver) String() string { @@ -91,8 +153,19 @@ func (d *Driver) Status() [][2]string { } } +// panicIfUsedByLcow does exactly what it says. +// TODO @jhowardmsft - this is a temporary measure for the bring-up of +// Linux containers on Windows. It is a failsafe to ensure that the right +// graphdriver is used. +func panicIfUsedByLcow() { + if system.LCOWSupported() { + panic("inconsistency - windowsfilter graphdriver should not be used when in LCOW mode") + } +} + // Exists returns true if the given id is registered with this driver. func (d *Driver) Exists(id string) bool { + panicIfUsedByLcow() rID, err := d.resolveID(id) if err != nil { return false @@ -106,20 +179,24 @@ func (d *Driver) Exists(id string) bool { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.create(id, parent, mountLabel, false, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + panicIfUsedByLcow() + if opts != nil { + return d.create(id, parent, opts.MountLabel, false, opts.StorageOpt) + } + return d.create(id, parent, "", false, nil) } // Create creates a new read-only layer with the given id. -func (d *Driver) Create(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.create(id, parent, mountLabel, true, storageOpt) +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + panicIfUsedByLcow() + if opts != nil { + return d.create(id, parent, opts.MountLabel, true, opts.StorageOpt) + } + return d.create(id, parent, "", true, nil) } func (d *Driver) create(id, parent, mountLabel string, readOnly bool, storageOpt map[string]string) error { - if len(storageOpt) != 0 { - return fmt.Errorf("--storage-opt is not supported for windows") - } - rPId, err := d.resolveID(parent) if err != nil { return err @@ -133,7 +210,7 @@ func (d *Driver) create(id, parent, mountLabel string, readOnly bool, storageOpt var layerChain []string if rPId != "" { - parentPath, err := hcsshim.LayerMountPath(d.info, rPId) + parentPath, err := hcsshim.GetLayerMountPath(d.info, rPId) if err != nil { return err } @@ -156,32 +233,20 @@ func (d *Driver) create(id, parent, mountLabel string, readOnly bool, storageOpt parentPath = layerChain[0] } - if isTP5OrOlder() { - // Pre-create the layer directory, providing an ACL to give the Hyper-V Virtual Machines - // group access. This is necessary to ensure that Hyper-V containers can access the - // virtual machine data. This is not necessary post-TP5. - path, err := syscall.UTF16FromString(filepath.Join(d.info.HomeDir, id)) - if err != nil { - return err - } - // Give system and administrators full control, and VMs read, write, and execute. - // Mark these ACEs as inherited. - sd, err := winio.SddlToSecurityDescriptor("D:(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FRFWFX;;;S-1-5-83-0)") - if err != nil { - return err - } - err = syscall.CreateDirectory(&path[0], &syscall.SecurityAttributes{ - Length: uint32(unsafe.Sizeof(syscall.SecurityAttributes{})), - SecurityDescriptor: uintptr(unsafe.Pointer(&sd[0])), - }) - if err != nil { - return err - } - } - if err := hcsshim.CreateSandboxLayer(d.info, id, parentPath, layerChain); err != nil { return err } + + storageOptions, err := parseStorageOpt(storageOpt) + if err != nil { + return fmt.Errorf("Failed to parse storage options - %s", err) + } + + if storageOptions.size != 0 { + if err := hcsshim.ExpandSandboxSize(d.info, id, storageOptions.size); err != nil { + return err + } + } } if _, err := os.Lstat(d.dir(parent)); err != nil { @@ -208,16 +273,89 @@ func (d *Driver) dir(id string) string { // Remove unmounts and removes the dir information. func (d *Driver) Remove(id string) error { + panicIfUsedByLcow() rID, err := d.resolveID(id) if err != nil { return err } - os.RemoveAll(filepath.Join(d.info.HomeDir, "sysfile-backups", rID)) // ok to fail - return hcsshim.DestroyLayer(d.info, rID) + + // This retry loop is due to a bug in Windows (Internal bug #9432268) + // if GetContainers fails with ErrVmcomputeOperationInvalidState + // it is a transient error. Retry until it succeeds. + var computeSystems []hcsshim.ContainerProperties + retryCount := 0 + osv := system.GetOSVersion() + for { + // Get and terminate any template VMs that are currently using the layer. + // Note: It is unfortunate that we end up in the graphdrivers Remove() call + // for both containers and images, but the logic for template VMs is only + // needed for images - specifically we are looking to see if a base layer + // is in use by a template VM as a result of having started a Hyper-V + // container at some point. + // + // We have a retry loop for ErrVmcomputeOperationInvalidState and + // ErrVmcomputeOperationAccessIsDenied as there is a race condition + // in RS1 and RS2 building during enumeration when a silo is going away + // for example under it, in HCS. AccessIsDenied added to fix 30278. + // + // TODO @jhowardmsft - For RS3, we can remove the retries. Also consider + // using platform APIs (if available) to get this more succinctly. Also + // consider enhancing the Remove() interface to have context of why + // the remove is being called - that could improve efficiency by not + // enumerating compute systems during a remove of a container as it's + // not required. + computeSystems, err = hcsshim.GetContainers(hcsshim.ComputeSystemQuery{}) + if err != nil { + if (osv.Build < 15139) && + ((err == hcsshim.ErrVmcomputeOperationInvalidState) || (err == hcsshim.ErrVmcomputeOperationAccessIsDenied)) { + if retryCount >= 500 { + break + } + retryCount++ + time.Sleep(10 * time.Millisecond) + continue + } + return err + } + break + } + + for _, computeSystem := range computeSystems { + if strings.Contains(computeSystem.RuntimeImagePath, id) && computeSystem.IsRuntimeTemplate { + container, err := hcsshim.OpenContainer(computeSystem.ID) + if err != nil { + return err + } + defer container.Close() + err = container.Terminate() + if hcsshim.IsPending(err) { + err = container.Wait() + } else if hcsshim.IsAlreadyStopped(err) { + err = nil + } + + if err != nil { + return err + } + } + } + + layerPath := filepath.Join(d.info.HomeDir, rID) + tmpID := fmt.Sprintf("%s-removing", rID) + tmpLayerPath := filepath.Join(d.info.HomeDir, tmpID) + if err := os.Rename(layerPath, tmpLayerPath); err != nil && !os.IsNotExist(err) { + return err + } + if err := hcsshim.DestroyLayer(d.info, tmpID); err != nil { + logrus.Errorf("Failed to DestroyLayer %s: %s", id, err) + } + + return nil } -// Get returns the rootfs path for the id. This will mount the dir at it's given path. +// Get returns the rootfs path for the id. This will mount the dir at its given path. func (d *Driver) Get(id, mountLabel string) (string, error) { + panicIfUsedByLcow() logrus.Debugf("WindowsGraphDriver Get() id %s mountLabel %s", id, mountLabel) var dir string @@ -248,9 +386,12 @@ func (d *Driver) Get(id, mountLabel string) (string, error) { return "", err } - mountPath, err := hcsshim.LayerMountPath(d.info, rID) + mountPath, err := hcsshim.GetLayerMountPath(d.info, rID) if err != nil { d.ctr.Decrement(rID) + if err := hcsshim.UnprepareLayer(d.info, rID); err != nil { + logrus.Warnf("Failed to Unprepare %s: %s", id, err) + } if err2 := hcsshim.DeactivateLayer(d.info, rID); err2 != nil { logrus.Warnf("Failed to Deactivate %s: %s", id, err) } @@ -273,6 +414,7 @@ func (d *Driver) Get(id, mountLabel string) (string, error) { // Put adds a new layer to the driver. func (d *Driver) Put(id string) error { + panicIfUsedByLcow() logrus.Debugf("WindowsGraphDriver Put() id %s", id) rID, err := d.resolveID(id) @@ -283,9 +425,15 @@ func (d *Driver) Put(id string) error { return nil } d.cacheMu.Lock() + _, exists := d.cache[rID] delete(d.cache, rID) d.cacheMu.Unlock() + // If the cache was not populated, then the layer was left unprepared and deactivated + if !exists { + return nil + } + if err := hcsshim.UnprepareLayer(d.info, rID); err != nil { return err } @@ -293,7 +441,31 @@ func (d *Driver) Put(id string) error { } // Cleanup ensures the information the driver stores is properly removed. +// We use this opportunity to cleanup any -removing folders which may be +// still left if the daemon was killed while it was removing a layer. func (d *Driver) Cleanup() error { + items, err := ioutil.ReadDir(d.info.HomeDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + // Note we don't return an error below - it's possible the files + // are locked. However, next time around after the daemon exits, + // we likely will be able to to cleanup successfully. Instead we log + // warnings if there are errors. + for _, item := range items { + if item.IsDir() && strings.HasSuffix(item.Name(), "-removing") { + if err := hcsshim.DestroyLayer(d.info, item.Name()); err != nil { + logrus.Warnf("Failed to cleanup %s: %s", item.Name(), err) + } else { + logrus.Infof("Cleaned up %s", item.Name()) + } + } + } + return nil } @@ -301,6 +473,7 @@ func (d *Driver) Cleanup() error { // layer and its parent layer which may be "". // The layer should be mounted when calling this function func (d *Driver) Diff(id, parent string) (_ io.ReadCloser, err error) { + panicIfUsedByLcow() rID, err := d.resolveID(id) if err != nil { return @@ -335,8 +508,9 @@ func (d *Driver) Diff(id, parent string) (_ io.ReadCloser, err error) { // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. -// The layer should be mounted when calling this function +// The layer should not be mounted when calling this function. func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { + panicIfUsedByLcow() rID, err := d.resolveID(id) if err != nil { return nil, err @@ -346,13 +520,12 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { return nil, err } - // this is assuming that the layer is unmounted - if err := hcsshim.UnprepareLayer(d.info, rID); err != nil { + if err := hcsshim.ActivateLayer(d.info, rID); err != nil { return nil, err } defer func() { - if err := hcsshim.PrepareLayer(d.info, rID, parentChain); err != nil { - logrus.Warnf("Failed to Deactivate %s: %s", rID, err) + if err2 := hcsshim.DeactivateLayer(d.info, rID); err2 != nil { + logrus.Errorf("changes() failed to DeactivateLayer %s %s: %s", id, rID, err2) } }() @@ -392,7 +565,8 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { // layer with the specified id and parent, returning the size of the // new layer in bytes. // The layer should not be mounted when calling this function -func (d *Driver) ApplyDiff(id, parent string, diff archive.Reader) (int64, error) { +func (d *Driver) ApplyDiff(id, parent string, diff io.Reader) (int64, error) { + panicIfUsedByLcow() var layerChain []string if parent != "" { rPId, err := d.resolveID(parent) @@ -403,7 +577,7 @@ func (d *Driver) ApplyDiff(id, parent string, diff archive.Reader) (int64, error if err != nil { return 0, err } - parentPath, err := hcsshim.LayerMountPath(d.info, rPId) + parentPath, err := hcsshim.GetLayerMountPath(d.info, rPId) if err != nil { return 0, err } @@ -427,6 +601,7 @@ func (d *Driver) ApplyDiff(id, parent string, diff archive.Reader) (int64, error // and its parent and returns the size in bytes of the changes // relative to its base filesystem directory. func (d *Driver) DiffSize(id, parent string) (size int64, err error) { + panicIfUsedByLcow() rPId, err := d.resolveID(parent) if err != nil { return @@ -448,6 +623,7 @@ func (d *Driver) DiffSize(id, parent string) (size int64, err error) { // Metadata returns custom driver information. func (d *Driver) Metadata(id string) (map[string]string, error) { + panicIfUsedByLcow() m := make(map[string]string) m["dir"] = d.dir(id) return m, nil @@ -505,7 +681,48 @@ func (d *Driver) exportLayer(id string, parentLayerPaths []string) (io.ReadClose return archive, nil } -func writeLayerFromTar(r archive.Reader, w hcsshim.LayerWriter) (int64, error) { +// writeBackupStreamFromTarAndSaveMutatedFiles reads data from a tar stream and +// writes it to a backup stream, and also saves any files that will be mutated +// by the import layer process to a backup location. +func writeBackupStreamFromTarAndSaveMutatedFiles(buf *bufio.Writer, w io.Writer, t *tar.Reader, hdr *tar.Header, root string) (nextHdr *tar.Header, err error) { + var bcdBackup *os.File + var bcdBackupWriter *winio.BackupFileWriter + if backupPath, ok := mutatedFiles[hdr.Name]; ok { + bcdBackup, err = os.Create(filepath.Join(root, backupPath)) + if err != nil { + return nil, err + } + defer func() { + cerr := bcdBackup.Close() + if err == nil { + err = cerr + } + }() + + bcdBackupWriter = winio.NewBackupFileWriter(bcdBackup, false) + defer func() { + cerr := bcdBackupWriter.Close() + if err == nil { + err = cerr + } + }() + + buf.Reset(io.MultiWriter(w, bcdBackupWriter)) + } else { + buf.Reset(w) + } + + defer func() { + ferr := buf.Flush() + if err == nil { + err = ferr + } + }() + + return backuptar.WriteBackupStreamFromTarFile(buf, t, hdr) +} + +func writeLayerFromTar(r io.Reader, w hcsshim.LayerWriter, root string) (int64, error) { t := tar.NewReader(r) hdr, err := t.Next() totalSize := int64(0) @@ -539,30 +756,7 @@ func writeLayerFromTar(r archive.Reader, w hcsshim.LayerWriter) (int64, error) { if err != nil { return 0, err } - buf.Reset(w) - - // Add the Hyper-V Virtual Machine group ACE to the security descriptor - // for TP5 so that Xenons can access all files. This is not necessary - // for post-TP5 builds. - if isTP5OrOlder() { - if sddl, ok := hdr.Winheaders["sd"]; ok { - var ace string - if hdr.Typeflag == tar.TypeDir { - ace = "(A;OICI;0x1200a9;;;S-1-5-83-0)" - } else { - ace = "(A;;0x1200a9;;;S-1-5-83-0)" - } - if hdr.Winheaders["sd"], ok = addAceToSddlDacl(sddl, ace); !ok { - logrus.Debugf("failed to add VM ACE to %s", sddl) - } - } - } - - hdr, err = backuptar.WriteBackupStreamFromTarFile(buf, t, hdr) - ferr := buf.Flush() - if ferr != nil { - err = ferr - } + hdr, err = writeBackupStreamFromTarAndSaveMutatedFiles(buf, w, t, hdr, root) totalSize += size } } @@ -572,105 +766,75 @@ func writeLayerFromTar(r archive.Reader, w hcsshim.LayerWriter) (int64, error) { return totalSize, nil } -func addAceToSddlDacl(sddl, ace string) (string, bool) { - daclStart := strings.Index(sddl, "D:") - if daclStart < 0 { - return sddl, false - } - - dacl := sddl[daclStart:] - daclEnd := strings.Index(dacl, "S:") - if daclEnd < 0 { - daclEnd = len(dacl) - } - dacl = dacl[:daclEnd] - - if strings.Contains(dacl, ace) { - return sddl, true - } - - i := 2 - for i+1 < len(dacl) { - if dacl[i] != '(' { - return sddl, false - } - - if dacl[i+1] == 'A' { - break - } - - i += 2 - for p := 1; i < len(dacl) && p > 0; i++ { - if dacl[i] == '(' { - p++ - } else if dacl[i] == ')' { - p-- - } - } - } - - return sddl[:daclStart+i] + ace + sddl[daclStart+i:], true -} - // importLayer adds a new layer to the tag and graph store based on the given data. -func (d *Driver) importLayer(id string, layerData archive.Reader, parentLayerPaths []string) (size int64, err error) { - cmd := reexec.Command(append([]string{"storage-windows-write-layer", d.info.HomeDir, id}, parentLayerPaths...)...) - output := bytes.NewBuffer(nil) - cmd.Stdin = layerData - cmd.Stdout = output - cmd.Stderr = output +func (d *Driver) importLayer(id string, layerData io.Reader, parentLayerPaths []string) (size int64, err error) { + if !noreexec { + cmd := reexec.Command(append([]string{"docker-windows-write-layer", d.info.HomeDir, id}, parentLayerPaths...)...) + output := bytes.NewBuffer(nil) + cmd.Stdin = layerData + cmd.Stdout = output + cmd.Stderr = output - if err = cmd.Start(); err != nil { - return + if err = cmd.Start(); err != nil { + return + } + + if err = cmd.Wait(); err != nil { + return 0, fmt.Errorf("re-exec error: %v: output: %s", err, output) + } + + return strconv.ParseInt(output.String(), 10, 64) } - - if err = cmd.Wait(); err != nil { - return 0, fmt.Errorf("re-exec error: %v: output: %s", err, output) - } - - return strconv.ParseInt(output.String(), 10, 64) + return writeLayer(layerData, d.info.HomeDir, id, parentLayerPaths...) } -// writeLayer is the re-exec entry point for writing a layer from a tar file -func writeLayer() { - home := os.Args[1] - id := os.Args[2] - parentLayerPaths := os.Args[3:] - - err := func() error { - err := winio.EnableProcessPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}) - if err != nil { - return err - } - - info := hcsshim.DriverInfo{ - Flavour: filterDriver, - HomeDir: home, - } - - w, err := hcsshim.NewLayerWriter(info, id, parentLayerPaths) - if err != nil { - return err - } - - size, err := writeLayerFromTar(os.Stdin, w) - if err != nil { - return err - } - - err = w.Close() - if err != nil { - return err - } - - fmt.Fprint(os.Stdout, size) - return nil - }() - +// writeLayerReexec is the re-exec entry point for writing a layer from a tar file +func writeLayerReexec() { + size, err := writeLayer(os.Stdin, os.Args[1], os.Args[2], os.Args[3:]...) if err != nil { fmt.Fprint(os.Stderr, err) os.Exit(1) } + fmt.Fprint(os.Stdout, size) +} + +// writeLayer writes a layer from a tar file. +func writeLayer(layerData io.Reader, home string, id string, parentLayerPaths ...string) (int64, error) { + err := winio.EnableProcessPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}) + if err != nil { + return 0, err + } + if noreexec { + defer func() { + if err := winio.DisableProcessPrivileges([]string{winio.SeBackupPrivilege, winio.SeRestorePrivilege}); err != nil { + // This should never happen, but just in case when in debugging mode. + // See https://github.com/docker/docker/pull/28002#discussion_r86259241 for rationale. + panic("Failed to disabled process privileges while in non re-exec mode") + } + }() + } + + info := hcsshim.DriverInfo{ + Flavour: filterDriver, + HomeDir: home, + } + + w, err := hcsshim.NewLayerWriter(info, id, parentLayerPaths) + if err != nil { + return 0, err + } + + size, err := writeLayerFromTar(layerData, w, filepath.Join(home, id)) + if err != nil { + return 0, err + } + + err = w.Close() + if err != nil { + return 0, err + } + + return size, nil } // resolveID computes the layerID information based on the given id. @@ -686,11 +850,7 @@ func (d *Driver) resolveID(id string) (string, error) { // setID stores the layerId in disk. func (d *Driver) setID(id, altID string) error { - err := ioutil.WriteFile(filepath.Join(d.dir(id), "layerId"), []byte(altID), 0600) - if err != nil { - return err - } - return nil + return ioutil.WriteFile(filepath.Join(d.dir(id), "layerId"), []byte(altID), 0600) } // getLayerChain returns the layer chain information. @@ -733,17 +893,23 @@ type fileGetCloserWithBackupPrivileges struct { } func (fg *fileGetCloserWithBackupPrivileges) Get(filename string) (io.ReadCloser, error) { + if backupPath, ok := mutatedFiles[filename]; ok { + return os.Open(filepath.Join(fg.path, backupPath)) + } + var f *os.File // Open the file while holding the Windows backup privilege. This ensures that the // file can be opened even if the caller does not actually have access to it according - // to the security descriptor. + // to the security descriptor. Also use sequential file access to avoid depleting the + // standby list - Microsoft VSO Bug Tracker #9900466 err := winio.RunWithPrivilege(winio.SeBackupPrivilege, func() error { path := longpath.AddPrefix(filepath.Join(fg.path, filename)) - p, err := syscall.UTF16FromString(path) + p, err := windows.UTF16FromString(path) if err != nil { return err } - h, err := syscall.CreateFile(&p[0], syscall.GENERIC_READ, syscall.FILE_SHARE_READ, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + const fileFlagSequentialScan = 0x08000000 // FILE_FLAG_SEQUENTIAL_SCAN + h, err := windows.CreateFile(&p[0], windows.GENERIC_READ, windows.FILE_SHARE_READ, nil, windows.OPEN_EXISTING, windows.FILE_FLAG_BACKUP_SEMANTICS|fileFlagSequentialScan, 0) if err != nil { return &os.PathError{Op: "open", Path: path, Err: err} } @@ -757,19 +923,10 @@ func (fg *fileGetCloserWithBackupPrivileges) Close() error { return nil } -type fileGetDestroyCloser struct { - storage.FileGetter - path string -} - -func (f *fileGetDestroyCloser) Close() error { - // TODO: activate layers and release here? - return os.RemoveAll(f.path) -} - // DiffGetter returns a FileGetCloser that can read files from the directory that // contains files for the layer differences. Used for direct access for tar-split. func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { + panicIfUsedByLcow() id, err := d.resolveID(id) if err != nil { return nil, err @@ -777,3 +934,32 @@ func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { return &fileGetCloserWithBackupPrivileges{d.dir(id)}, nil } + +// AdditionalImageStores returns additional image stores supported by the driver +func (d *Driver) AdditionalImageStores() []string { + return nil +} + +type storageOptions struct { + size uint64 +} + +func parseStorageOpt(storageOpt map[string]string) (*storageOptions, error) { + options := storageOptions{} + + // Read size to change the block device size per container. + for key, val := range storageOpt { + key := strings.ToLower(key) + switch key { + case "size": + size, err := units.RAMInBytes(val) + if err != nil { + return nil, err + } + options.size = uint64(size) + default: + return nil, fmt.Errorf("Unknown storage option: %s", key) + } + } + return &options, nil +} diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs.go b/vendor/github.com/containers/storage/drivers/zfs/zfs.go index c9860ec2..8c8e7d67 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" "sync" - "syscall" "time" "github.com/containers/storage/drivers" @@ -21,6 +20,7 @@ import ( "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) type zfsOptions struct { @@ -100,6 +100,14 @@ func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdri return nil, fmt.Errorf("BUG: zfs get all -t filesystem -rHp '%s' should contain '%s'", options.fsName, options.fsName) } + rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) + if err != nil { + return nil, fmt.Errorf("Failed to get root uid/guid: %v", err) + } + if err := idtools.MkdirAllAs(base, 0700, rootUID, rootGID); err != nil { + return nil, fmt.Errorf("Failed to create '%s': %v", base, err) + } + if err := mount.MakePrivate(base); err != nil { return nil, err } @@ -134,8 +142,8 @@ func parseOptions(opt []string) (zfsOptions, error) { } func lookupZfsDataset(rootdir string) (string, error) { - var stat syscall.Stat_t - if err := syscall.Stat(rootdir, &stat); err != nil { + var stat unix.Stat_t + if err := unix.Stat(rootdir, &stat); err != nil { return "", fmt.Errorf("Failed to access '%s': %s", rootdir, err) } wantedDev := stat.Dev @@ -145,7 +153,7 @@ func lookupZfsDataset(rootdir string) (string, error) { return "", err } for _, m := range mounts { - if err := syscall.Stat(m.Mountpoint, &stat); err != nil { + if err := unix.Stat(m.Mountpoint, &stat); err != nil { logrus.Debugf("[zfs] failed to stat '%s' while scanning for zfs mount: %v", m.Mountpoint, err) continue // may fail on fuse file systems } @@ -213,7 +221,10 @@ func (d *Driver) Status() [][2]string { // Metadata returns image/container metadata related to graph driver func (d *Driver) Metadata(id string) (map[string]string, error) { - return nil, nil + return map[string]string{ + "Mountpoint": d.mountPath(id), + "Dataset": d.zfsPath(id), + }, nil } func (d *Driver) cloneFilesystem(name, parentName string) error { @@ -248,12 +259,17 @@ func (d *Driver) mountPath(id string) string { // CreateReadWrite creates a layer that is writable for use as a container // file system. -func (d *Driver) CreateReadWrite(id, parent, mountLabel string, storageOpt map[string]string) error { - return d.Create(id, parent, mountLabel, storageOpt) +func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { + return d.Create(id, parent, opts) } // Create prepares the dataset and filesystem for the ZFS driver for the given id under the parent. -func (d *Driver) Create(id string, parent string, mountLabel string, storageOpt map[string]string) error { +func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + var storageOpt map[string]string + if opts != nil { + storageOpt = opts.StorageOpt + } + err := d.create(id, parent, storageOpt) if err == nil { return nil @@ -391,22 +407,20 @@ func (d *Driver) Put(id string) error { logrus.Debugf(`[zfs] unmount("%s")`, mountpoint) - err = mount.Unmount(mountpoint) - if err != nil { + if err := mount.Unmount(mountpoint); err != nil { return fmt.Errorf("error unmounting to %s: %v", mountpoint, err) } - return err + return nil } // Exists checks to see if the cache entry exists for the given id. func (d *Driver) Exists(id string) bool { d.Lock() defer d.Unlock() - return d.filesystemsCache[d.zfsPath(id)] == true + return d.filesystemsCache[d.zfsPath(id)] } // AdditionalImageStores returns additional image stores supported by the driver func (d *Driver) AdditionalImageStores() []string { - var imageStores []string - return imageStores + return nil } diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs_freebsd.go b/vendor/github.com/containers/storage/drivers/zfs/zfs_freebsd.go index ade71b15..69c0448d 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs_freebsd.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs_freebsd.go @@ -3,16 +3,16 @@ package zfs import ( "fmt" "strings" - "syscall" "github.com/containers/storage/drivers" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) func checkRootdirFs(rootdir string) error { - var buf syscall.Statfs_t - if err := syscall.Statfs(rootdir, &buf); err != nil { + var buf unix.Statfs_t + if err := unix.Statfs(rootdir, &buf); err != nil { return fmt.Errorf("Failed to access '%s': %s", rootdir, err) } diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs_linux.go b/vendor/github.com/containers/storage/drivers/zfs/zfs_linux.go index 92b38756..da298047 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs_linux.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs_linux.go @@ -2,16 +2,16 @@ package zfs import ( "fmt" - "syscall" "github.com/containers/storage/drivers" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) func checkRootdirFs(rootdir string) error { - var buf syscall.Statfs_t - if err := syscall.Statfs(rootdir, &buf); err != nil { + var buf unix.Statfs_t + if err := unix.Statfs(rootdir, &buf); err != nil { return fmt.Errorf("Failed to access '%s': %s", rootdir, err) } diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs_solaris.go b/vendor/github.com/containers/storage/drivers/zfs/zfs_solaris.go index ca595638..2383bf3b 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs_solaris.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs_solaris.go @@ -22,24 +22,23 @@ import ( "github.com/containers/storage/drivers" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus" ) func checkRootdirFs(rootdir string) error { cs := C.CString(filepath.Dir(rootdir)) + defer C.free(unsafe.Pointer(cs)) buf := C.getstatfs(cs) + defer C.free(unsafe.Pointer(buf)) // on Solaris buf.f_basetype contains ['z', 'f', 's', 0 ... ] if (buf.f_basetype[0] != 122) || (buf.f_basetype[1] != 102) || (buf.f_basetype[2] != 115) || (buf.f_basetype[3] != 0) { - log.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir) - C.free(unsafe.Pointer(buf)) + logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir) return errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", rootdir) } - C.free(unsafe.Pointer(buf)) - C.free(unsafe.Pointer(cs)) return nil } diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index effb57f4..b2129b42 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -24,8 +24,9 @@ type Image struct { // unique among images. Names []string `json:"names,omitempty"` - // TopLayer is the ID of the topmost layer of the image itself. - // Multiple images can refer to the same top layer. + // TopLayer is the ID of the topmost layer of the image itself, if the + // image contains one or more layers. Multiple images can refer to the + // same top layer. TopLayer string `json:"layer"` // Metadata is data we keep for the convenience of the caller. It is not @@ -270,6 +271,7 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string, c if _, idInUse := r.byid[id]; idInUse { return nil, ErrDuplicateID } + names = dedupeNames(names) for _, name := range names { if _, nameInUse := r.byname[name]; nameInUse { return nil, ErrDuplicateName @@ -326,6 +328,7 @@ func (r *imageStore) SetNames(id string, names []string) error { if !r.IsReadWrite() { return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to change image name assignments at %q", r.imagespath()) } + names = dedupeNames(names) if image, ok := r.lookup(id); ok { for _, name := range image.Names { delete(r.byname, name) diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 90481ec7..105875d0 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -490,15 +490,20 @@ func (r *layerStore) Put(id, parent string, names []string, mountLabel string, o if _, idInUse := r.byid[id]; idInUse { return nil, -1, ErrDuplicateID } + names = dedupeNames(names) for _, name := range names { if _, nameInUse := r.byname[name]; nameInUse { return nil, -1, ErrDuplicateName } } + opts := drivers.CreateOpts{ + MountLabel: mountLabel, + StorageOpt: options, + } if writeable { - err = r.driver.CreateReadWrite(id, parent, mountLabel, options) + err = r.driver.CreateReadWrite(id, parent, &opts) } else { - err = r.driver.Create(id, parent, mountLabel, options) + err = r.driver.Create(id, parent, &opts) } if err == nil { layer = &Layer{ @@ -622,6 +627,7 @@ func (r *layerStore) SetNames(id string, names []string) error { if !r.IsReadWrite() { return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to change layer name assignments at %q", r.layerspath()) } + names = dedupeNames(names) if layer, ok := r.lookup(id); ok { for _, name := range layer.Names { delete(r.byname, name) diff --git a/vendor/github.com/containers/storage/pkg/archive/example_changes.go b/vendor/github.com/containers/storage/pkg/archive/example_changes.go new file mode 100644 index 00000000..70f9c556 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/example_changes.go @@ -0,0 +1,97 @@ +// +build ignore + +// Simple tool to create an archive stream from an old and new directory +// +// By default it will stream the comparison of two temporary directories with junk files +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/containers/storage/pkg/archive" + "github.com/sirupsen/logrus" +) + +var ( + flDebug = flag.Bool("D", false, "debugging output") + flNewDir = flag.String("newdir", "", "") + flOldDir = flag.String("olddir", "", "") + log = logrus.New() +) + +func main() { + flag.Usage = func() { + fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)") + fmt.Printf("%s [OPTIONS]\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + log.Out = os.Stderr + if (len(os.Getenv("DEBUG")) > 0) || *flDebug { + logrus.SetLevel(logrus.DebugLevel) + } + var newDir, oldDir string + + if len(*flNewDir) == 0 { + var err error + newDir, err = ioutil.TempDir("", "storage-test-newDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(newDir) + if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil { + log.Fatal(err) + } + } else { + newDir = *flNewDir + } + + if len(*flOldDir) == 0 { + oldDir, err := ioutil.TempDir("", "storage-test-oldDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(oldDir) + } else { + oldDir = *flOldDir + } + + changes, err := archive.ChangesDirs(newDir, oldDir) + if err != nil { + log.Fatal(err) + } + + a, err := archive.ExportChanges(newDir, changes) + if err != nil { + log.Fatal(err) + } + defer a.Close() + + i, err := io.Copy(os.Stdout, a) + if err != nil && err != io.EOF { + log.Fatal(err) + } + fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i) +} + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + if makeLinks { + if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { + return 0, err + } + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} diff --git a/vendor/github.com/containers/storage/pkg/dmesg/dmesg_linux.go b/vendor/github.com/containers/storage/pkg/dmesg/dmesg_linux.go new file mode 100644 index 00000000..7df7f3d4 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/dmesg/dmesg_linux.go @@ -0,0 +1,20 @@ +// +build linux + +package dmesg + +import ( + "unsafe" + + "golang.org/x/sys/unix" +) + +// Dmesg returns last messages from the kernel log, up to size bytes +func Dmesg(size int) []byte { + t := uintptr(3) // SYSLOG_ACTION_READ_ALL + b := make([]byte, size) + amt, _, err := unix.Syscall(unix.SYS_SYSLOG, t, uintptr(unsafe.Pointer(&b[0])), uintptr(len(b))) + if err != 0 { + return []byte{} + } + return b[:amt] +} diff --git a/vendor/github.com/containers/storage/pkg/fsutils/fsutils_linux.go b/vendor/github.com/containers/storage/pkg/fsutils/fsutils_linux.go new file mode 100644 index 00000000..e6094b55 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/fsutils/fsutils_linux.go @@ -0,0 +1,88 @@ +// +build linux + +package fsutils + +import ( + "fmt" + "io/ioutil" + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +func locateDummyIfEmpty(path string) (string, error) { + children, err := ioutil.ReadDir(path) + if err != nil { + return "", err + } + if len(children) != 0 { + return "", nil + } + dummyFile, err := ioutil.TempFile(path, "fsutils-dummy") + if err != nil { + return "", err + } + name := dummyFile.Name() + err = dummyFile.Close() + return name, err +} + +// SupportsDType returns whether the filesystem mounted on path supports d_type +func SupportsDType(path string) (bool, error) { + // locate dummy so that we have at least one dirent + dummy, err := locateDummyIfEmpty(path) + if err != nil { + return false, err + } + if dummy != "" { + defer os.Remove(dummy) + } + + visited := 0 + supportsDType := true + fn := func(ent *unix.Dirent) bool { + visited++ + if ent.Type == unix.DT_UNKNOWN { + supportsDType = false + // stop iteration + return true + } + // continue iteration + return false + } + if err = iterateReadDir(path, fn); err != nil { + return false, err + } + if visited == 0 { + return false, fmt.Errorf("did not hit any dirent during iteration %s", path) + } + return supportsDType, nil +} + +func iterateReadDir(path string, fn func(*unix.Dirent) bool) error { + d, err := os.Open(path) + if err != nil { + return err + } + defer d.Close() + fd := int(d.Fd()) + buf := make([]byte, 4096) + for { + nbytes, err := unix.ReadDirent(fd, buf) + if err != nil { + return err + } + if nbytes == 0 { + break + } + for off := 0; off < nbytes; { + ent := (*unix.Dirent)(unsafe.Pointer(&buf[off])) + if stop := fn(ent); stop { + return nil + } + off += int(ent.Reclen) + } + } + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/locker/README.md b/vendor/github.com/containers/storage/pkg/locker/README.md new file mode 100644 index 00000000..ad15e89a --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/locker/README.md @@ -0,0 +1,65 @@ +Locker +===== + +locker provides a mechanism for creating finer-grained locking to help +free up more global locks to handle other tasks. + +The implementation looks close to a sync.Mutex, however, the user must provide a +reference to use to refer to the underlying lock when locking and unlocking, +and unlock may generate an error. + +If a lock with a given name does not exist when `Lock` is called, one is +created. +Lock references are automatically cleaned up on `Unlock` if nothing else is +waiting for the lock. + + +## Usage + +```go +package important + +import ( + "sync" + "time" + + "github.com/containers/storage/pkg/locker" +) + +type important struct { + locks *locker.Locker + data map[string]interface{} + mu sync.Mutex +} + +func (i *important) Get(name string) interface{} { + i.locks.Lock(name) + defer i.locks.Unlock(name) + return data[name] +} + +func (i *important) Create(name string, data interface{}) { + i.locks.Lock(name) + defer i.locks.Unlock(name) + + i.createImportant(data) + + s.mu.Lock() + i.data[name] = data + s.mu.Unlock() +} + +func (i *important) createImportant(data interface{}) { + time.Sleep(10 * time.Second) +} +``` + +For functions dealing with a given name, always lock at the beginning of the +function (or before doing anything with the underlying state), this ensures any +other function that is dealing with the same name will block. + +When needing to modify the underlying data, use the global lock to ensure nothing +else is modifying it at the same time. +Since name lock is already in place, no reads will occur while the modification +is being performed. + diff --git a/vendor/github.com/containers/storage/pkg/locker/locker.go b/vendor/github.com/containers/storage/pkg/locker/locker.go new file mode 100644 index 00000000..0b22ddfa --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/locker/locker.go @@ -0,0 +1,112 @@ +/* +Package locker provides a mechanism for creating finer-grained locking to help +free up more global locks to handle other tasks. + +The implementation looks close to a sync.Mutex, however the user must provide a +reference to use to refer to the underlying lock when locking and unlocking, +and unlock may generate an error. + +If a lock with a given name does not exist when `Lock` is called, one is +created. +Lock references are automatically cleaned up on `Unlock` if nothing else is +waiting for the lock. +*/ +package locker + +import ( + "errors" + "sync" + "sync/atomic" +) + +// ErrNoSuchLock is returned when the requested lock does not exist +var ErrNoSuchLock = errors.New("no such lock") + +// Locker provides a locking mechanism based on the passed in reference name +type Locker struct { + mu sync.Mutex + locks map[string]*lockCtr +} + +// lockCtr is used by Locker to represent a lock with a given name. +type lockCtr struct { + mu sync.Mutex + // waiters is the number of waiters waiting to acquire the lock + // this is int32 instead of uint32 so we can add `-1` in `dec()` + waiters int32 +} + +// inc increments the number of waiters waiting for the lock +func (l *lockCtr) inc() { + atomic.AddInt32(&l.waiters, 1) +} + +// dec decrements the number of waiters waiting on the lock +func (l *lockCtr) dec() { + atomic.AddInt32(&l.waiters, -1) +} + +// count gets the current number of waiters +func (l *lockCtr) count() int32 { + return atomic.LoadInt32(&l.waiters) +} + +// Lock locks the mutex +func (l *lockCtr) Lock() { + l.mu.Lock() +} + +// Unlock unlocks the mutex +func (l *lockCtr) Unlock() { + l.mu.Unlock() +} + +// New creates a new Locker +func New() *Locker { + return &Locker{ + locks: make(map[string]*lockCtr), + } +} + +// Lock locks a mutex with the given name. If it doesn't exist, one is created +func (l *Locker) Lock(name string) { + l.mu.Lock() + if l.locks == nil { + l.locks = make(map[string]*lockCtr) + } + + nameLock, exists := l.locks[name] + if !exists { + nameLock = &lockCtr{} + l.locks[name] = nameLock + } + + // increment the nameLock waiters while inside the main mutex + // this makes sure that the lock isn't deleted if `Lock` and `Unlock` are called concurrently + nameLock.inc() + l.mu.Unlock() + + // Lock the nameLock outside the main mutex so we don't block other operations + // once locked then we can decrement the number of waiters for this lock + nameLock.Lock() + nameLock.dec() +} + +// Unlock unlocks the mutex with the given name +// If the given lock is not being waited on by any other callers, it is deleted +func (l *Locker) Unlock(name string) error { + l.mu.Lock() + nameLock, exists := l.locks[name] + if !exists { + l.mu.Unlock() + return ErrNoSuchLock + } + + if nameLock.count() == 0 { + delete(l.locks, name) + } + nameLock.Unlock() + + l.mu.Unlock() + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/plugins/client.go b/vendor/github.com/containers/storage/pkg/plugins/client.go deleted file mode 100644 index 2c0a91d1..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/client.go +++ /dev/null @@ -1,205 +0,0 @@ -package plugins - -import ( - "bytes" - "encoding/json" - "io" - "io/ioutil" - "net/http" - "net/url" - "time" - - "github.com/containers/storage/pkg/plugins/transport" - "github.com/docker/go-connections/sockets" - "github.com/docker/go-connections/tlsconfig" - "github.com/sirupsen/logrus" -) - -const ( - defaultTimeOut = 30 -) - -func newTransport(addr string, tlsConfig *tlsconfig.Options) (transport.Transport, error) { - tr := &http.Transport{} - - if tlsConfig != nil { - c, err := tlsconfig.Client(*tlsConfig) - if err != nil { - return nil, err - } - tr.TLSClientConfig = c - } - - u, err := url.Parse(addr) - if err != nil { - return nil, err - } - socket := u.Host - if socket == "" { - // valid local socket addresses have the host empty. - socket = u.Path - } - if err := sockets.ConfigureTransport(tr, u.Scheme, socket); err != nil { - return nil, err - } - scheme := httpScheme(u) - - return transport.NewHTTPTransport(tr, scheme, socket), nil -} - -// NewClient creates a new plugin client (http). -func NewClient(addr string, tlsConfig *tlsconfig.Options) (*Client, error) { - clientTransport, err := newTransport(addr, tlsConfig) - if err != nil { - return nil, err - } - return newClientWithTransport(clientTransport, 0), nil -} - -// NewClientWithTimeout creates a new plugin client (http). -func NewClientWithTimeout(addr string, tlsConfig *tlsconfig.Options, timeout time.Duration) (*Client, error) { - clientTransport, err := newTransport(addr, tlsConfig) - if err != nil { - return nil, err - } - return newClientWithTransport(clientTransport, timeout), nil -} - -// newClientWithTransport creates a new plugin client with a given transport. -func newClientWithTransport(tr transport.Transport, timeout time.Duration) *Client { - return &Client{ - http: &http.Client{ - Transport: tr, - Timeout: timeout, - }, - requestFactory: tr, - } -} - -// Client represents a plugin client. -type Client struct { - http *http.Client // http client to use - requestFactory transport.RequestFactory -} - -// Call calls the specified method with the specified arguments for the plugin. -// It will retry for 30 seconds if a failure occurs when calling. -func (c *Client) Call(serviceMethod string, args interface{}, ret interface{}) error { - var buf bytes.Buffer - if args != nil { - if err := json.NewEncoder(&buf).Encode(args); err != nil { - return err - } - } - body, err := c.callWithRetry(serviceMethod, &buf, true) - if err != nil { - return err - } - defer body.Close() - if ret != nil { - if err := json.NewDecoder(body).Decode(&ret); err != nil { - logrus.Errorf("%s: error reading plugin resp: %v", serviceMethod, err) - return err - } - } - return nil -} - -// Stream calls the specified method with the specified arguments for the plugin and returns the response body -func (c *Client) Stream(serviceMethod string, args interface{}) (io.ReadCloser, error) { - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(args); err != nil { - return nil, err - } - return c.callWithRetry(serviceMethod, &buf, true) -} - -// SendFile calls the specified method, and passes through the IO stream -func (c *Client) SendFile(serviceMethod string, data io.Reader, ret interface{}) error { - body, err := c.callWithRetry(serviceMethod, data, true) - if err != nil { - return err - } - defer body.Close() - if err := json.NewDecoder(body).Decode(&ret); err != nil { - logrus.Errorf("%s: error reading plugin resp: %v", serviceMethod, err) - return err - } - return nil -} - -func (c *Client) callWithRetry(serviceMethod string, data io.Reader, retry bool) (io.ReadCloser, error) { - var retries int - start := time.Now() - - for { - req, err := c.requestFactory.NewRequest(serviceMethod, data) - if err != nil { - return nil, err - } - - resp, err := c.http.Do(req) - if err != nil { - if !retry { - return nil, err - } - - timeOff := backoff(retries) - if abort(start, timeOff) { - return nil, err - } - retries++ - logrus.Warnf("Unable to connect to plugin: %s%s: %v, retrying in %v", req.URL.Host, req.URL.Path, err, timeOff) - time.Sleep(timeOff) - continue - } - - if resp.StatusCode != http.StatusOK { - b, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, &statusError{resp.StatusCode, serviceMethod, err.Error()} - } - - // Plugins' Response(s) should have an Err field indicating what went - // wrong. Try to unmarshal into ResponseErr. Otherwise fallback to just - // return the string(body) - type responseErr struct { - Err string - } - remoteErr := responseErr{} - if err := json.Unmarshal(b, &remoteErr); err == nil { - if remoteErr.Err != "" { - return nil, &statusError{resp.StatusCode, serviceMethod, remoteErr.Err} - } - } - // old way... - return nil, &statusError{resp.StatusCode, serviceMethod, string(b)} - } - return resp.Body, nil - } -} - -func backoff(retries int) time.Duration { - b, max := 1, defaultTimeOut - for b < max && retries > 0 { - b *= 2 - retries-- - } - if b > max { - b = max - } - return time.Duration(b) * time.Second -} - -func abort(start time.Time, timeOff time.Duration) bool { - return timeOff+time.Since(start) >= time.Duration(defaultTimeOut)*time.Second -} - -func httpScheme(u *url.URL) string { - scheme := u.Scheme - if scheme != "https" { - scheme = "http" - } - return scheme -} diff --git a/vendor/github.com/containers/storage/pkg/plugins/discovery.go b/vendor/github.com/containers/storage/pkg/plugins/discovery.go deleted file mode 100644 index 8d8d2ead..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/discovery.go +++ /dev/null @@ -1,131 +0,0 @@ -package plugins - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/url" - "os" - "path/filepath" - "strings" - "sync" -) - -var ( - // ErrNotFound plugin not found - ErrNotFound = errors.New("plugin not found") - socketsPath = "/run/container/storage/plugins" -) - -// localRegistry defines a registry that is local (using unix socket). -type localRegistry struct{} - -func newLocalRegistry() localRegistry { - return localRegistry{} -} - -// Scan scans all the plugin paths and returns all the names it found -func Scan() ([]string, error) { - var names []string - if err := filepath.Walk(socketsPath, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return nil - } - - if fi.Mode()&os.ModeSocket != 0 { - name := strings.TrimSuffix(fi.Name(), filepath.Ext(fi.Name())) - names = append(names, name) - } - return nil - }); err != nil { - return nil, err - } - - for _, path := range specsPaths { - if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error { - if err != nil || fi.IsDir() { - return nil - } - name := strings.TrimSuffix(fi.Name(), filepath.Ext(fi.Name())) - names = append(names, name) - return nil - }); err != nil { - return nil, err - } - } - return names, nil -} - -// Plugin returns the plugin registered with the given name (or returns an error). -func (l *localRegistry) Plugin(name string) (*Plugin, error) { - socketpaths := pluginPaths(socketsPath, name, ".sock") - - for _, p := range socketpaths { - if fi, err := os.Stat(p); err == nil && fi.Mode()&os.ModeSocket != 0 { - return NewLocalPlugin(name, "unix://"+p), nil - } - } - - var txtspecpaths []string - for _, p := range specsPaths { - txtspecpaths = append(txtspecpaths, pluginPaths(p, name, ".spec")...) - txtspecpaths = append(txtspecpaths, pluginPaths(p, name, ".json")...) - } - - for _, p := range txtspecpaths { - if _, err := os.Stat(p); err == nil { - if strings.HasSuffix(p, ".json") { - return readPluginJSONInfo(name, p) - } - return readPluginInfo(name, p) - } - } - return nil, ErrNotFound -} - -func readPluginInfo(name, path string) (*Plugin, error) { - content, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - addr := strings.TrimSpace(string(content)) - - u, err := url.Parse(addr) - if err != nil { - return nil, err - } - - if len(u.Scheme) == 0 { - return nil, fmt.Errorf("Unknown protocol") - } - - return NewLocalPlugin(name, addr), nil -} - -func readPluginJSONInfo(name, path string) (*Plugin, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - var p Plugin - if err := json.NewDecoder(f).Decode(&p); err != nil { - return nil, err - } - p.name = name - if p.TLSConfig != nil && len(p.TLSConfig.CAFile) == 0 { - p.TLSConfig.InsecureSkipVerify = true - } - p.activateWait = sync.NewCond(&sync.Mutex{}) - - return &p, nil -} - -func pluginPaths(base, name, ext string) []string { - return []string{ - filepath.Join(base, name+ext), - filepath.Join(base, name, name+ext), - } -} diff --git a/vendor/github.com/containers/storage/pkg/plugins/discovery_unix.go b/vendor/github.com/containers/storage/pkg/plugins/discovery_unix.go deleted file mode 100644 index dd01df39..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/discovery_unix.go +++ /dev/null @@ -1,5 +0,0 @@ -// +build !windows - -package plugins - -var specsPaths = []string{"/etc/containers/storage/plugins", "/usr/lib/containers/storage/plugins"} diff --git a/vendor/github.com/containers/storage/pkg/plugins/discovery_windows.go b/vendor/github.com/containers/storage/pkg/plugins/discovery_windows.go deleted file mode 100644 index b4f55302..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/discovery_windows.go +++ /dev/null @@ -1,8 +0,0 @@ -package plugins - -import ( - "os" - "path/filepath" -) - -var specsPaths = []string{filepath.Join(os.Getenv("programdata"), "containers", "storage", "plugins")} diff --git a/vendor/github.com/containers/storage/pkg/plugins/errors.go b/vendor/github.com/containers/storage/pkg/plugins/errors.go deleted file mode 100644 index 79884710..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/errors.go +++ /dev/null @@ -1,33 +0,0 @@ -package plugins - -import ( - "fmt" - "net/http" -) - -type statusError struct { - status int - method string - err string -} - -// Error returns a formatted string for this error type -func (e *statusError) Error() string { - return fmt.Sprintf("%s: %v", e.method, e.err) -} - -// IsNotFound indicates if the passed in error is from an http.StatusNotFound from the plugin -func IsNotFound(err error) bool { - return isStatusError(err, http.StatusNotFound) -} - -func isStatusError(err error, status int) bool { - if err == nil { - return false - } - e, ok := err.(*statusError) - if !ok { - return false - } - return e.status == status -} diff --git a/vendor/github.com/containers/storage/pkg/plugins/plugins.go b/vendor/github.com/containers/storage/pkg/plugins/plugins.go deleted file mode 100644 index f7773b0a..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/plugins.go +++ /dev/null @@ -1,329 +0,0 @@ -// Package plugins provides structures and helper functions to manage Docker -// plugins. -// -// Docker discovers plugins by looking for them in the plugin directory whenever -// a user or container tries to use one by name. UNIX domain socket files must -// be located under /run/container/storage/plugins, whereas spec files can be located -// either under /etc/container/storage/plugins or /usr/lib/container/storage/plugins. This is handled -// by the Registry interface, which lets you list all plugins or get a plugin by -// its name if it exists. -// -// The plugins need to implement an HTTP server and bind this to the UNIX socket -// or the address specified in the spec files. -// A handshake is send at /Plugin.Activate, and plugins are expected to return -// a Manifest with a list of of Docker subsystems which this plugin implements. -// -// In order to use a plugins, you can use the ``Get`` with the name of the -// plugin and the subsystem it implements. -// -// plugin, err := plugins.Get("example", "VolumeDriver") -// if err != nil { -// return fmt.Errorf("Error looking up volume plugin example: %v", err) -// } -package plugins - -import ( - "errors" - "sync" - "time" - - "github.com/docker/go-connections/tlsconfig" - "github.com/sirupsen/logrus" -) - -var ( - // ErrNotImplements is returned if the plugin does not implement the requested driver. - ErrNotImplements = errors.New("Plugin does not implement the requested driver") -) - -type plugins struct { - sync.Mutex - plugins map[string]*Plugin -} - -type extpointHandlers struct { - sync.RWMutex - extpointHandlers map[string][]func(string, *Client) -} - -var ( - storage = plugins{plugins: make(map[string]*Plugin)} - handlers = extpointHandlers{extpointHandlers: make(map[string][]func(string, *Client))} -) - -// Manifest lists what a plugin implements. -type Manifest struct { - // List of subsystem the plugin implements. - Implements []string -} - -// Plugin is the definition of a container/storage plugin. -type Plugin struct { - // Name of the plugin - name string - // Address of the plugin - Addr string - // TLS configuration of the plugin - TLSConfig *tlsconfig.Options - // Client attached to the plugin - client *Client - // Manifest of the plugin (see above) - Manifest *Manifest `json:"-"` - - // wait for activation to finish - activateWait *sync.Cond - // error produced by activation - activateErr error - // keeps track of callback handlers run against this plugin - handlersRun bool -} - -// Name returns the name of the plugin. -func (p *Plugin) Name() string { - return p.name -} - -// Client returns a ready-to-use plugin client that can be used to communicate with the plugin. -func (p *Plugin) Client() *Client { - return p.client -} - -// IsV1 returns true for V1 plugins and false otherwise. -func (p *Plugin) IsV1() bool { - return true -} - -// NewLocalPlugin creates a new local plugin. -func NewLocalPlugin(name, addr string) *Plugin { - return &Plugin{ - name: name, - Addr: addr, - // TODO: change to nil - TLSConfig: &tlsconfig.Options{InsecureSkipVerify: true}, - activateWait: sync.NewCond(&sync.Mutex{}), - } -} - -func (p *Plugin) activate() error { - p.activateWait.L.Lock() - - if p.activated() { - p.runHandlers() - p.activateWait.L.Unlock() - return p.activateErr - } - - p.activateErr = p.activateWithLock() - - p.runHandlers() - p.activateWait.L.Unlock() - p.activateWait.Broadcast() - return p.activateErr -} - -// runHandlers runs the registered handlers for the implemented plugin types -// This should only be run after activation, and while the activation lock is held. -func (p *Plugin) runHandlers() { - if !p.activated() { - return - } - - handlers.RLock() - if !p.handlersRun { - for _, iface := range p.Manifest.Implements { - hdlrs, handled := handlers.extpointHandlers[iface] - if !handled { - continue - } - for _, handler := range hdlrs { - handler(p.name, p.client) - } - } - p.handlersRun = true - } - handlers.RUnlock() - -} - -// activated returns if the plugin has already been activated. -// This should only be called with the activation lock held -func (p *Plugin) activated() bool { - return p.Manifest != nil -} - -func (p *Plugin) activateWithLock() error { - c, err := NewClient(p.Addr, p.TLSConfig) - if err != nil { - return err - } - p.client = c - - m := new(Manifest) - if err = p.client.Call("Plugin.Activate", nil, m); err != nil { - return err - } - - p.Manifest = m - return nil -} - -func (p *Plugin) waitActive() error { - p.activateWait.L.Lock() - for !p.activated() && p.activateErr == nil { - p.activateWait.Wait() - } - p.activateWait.L.Unlock() - return p.activateErr -} - -func (p *Plugin) implements(kind string) bool { - if p.Manifest == nil { - return false - } - for _, driver := range p.Manifest.Implements { - if driver == kind { - return true - } - } - return false -} - -func load(name string) (*Plugin, error) { - return loadWithRetry(name, true) -} - -func loadWithRetry(name string, retry bool) (*Plugin, error) { - registry := newLocalRegistry() - start := time.Now() - - var retries int - for { - pl, err := registry.Plugin(name) - if err != nil { - if !retry { - return nil, err - } - - timeOff := backoff(retries) - if abort(start, timeOff) { - return nil, err - } - retries++ - logrus.Warnf("Unable to locate plugin: %s, retrying in %v", name, timeOff) - time.Sleep(timeOff) - continue - } - - storage.Lock() - if pl, exists := storage.plugins[name]; exists { - storage.Unlock() - return pl, pl.activate() - } - storage.plugins[name] = pl - storage.Unlock() - - err = pl.activate() - - if err != nil { - storage.Lock() - delete(storage.plugins, name) - storage.Unlock() - } - - return pl, err - } -} - -func get(name string) (*Plugin, error) { - storage.Lock() - pl, ok := storage.plugins[name] - storage.Unlock() - if ok { - return pl, pl.activate() - } - return load(name) -} - -// Get returns the plugin given the specified name and requested implementation. -func Get(name, imp string) (*Plugin, error) { - pl, err := get(name) - if err != nil { - return nil, err - } - if err := pl.waitActive(); err == nil && pl.implements(imp) { - logrus.Debugf("%s implements: %s", name, imp) - return pl, nil - } - return nil, ErrNotImplements -} - -// Handle adds the specified function to the extpointHandlers. -func Handle(iface string, fn func(string, *Client)) { - handlers.Lock() - hdlrs, ok := handlers.extpointHandlers[iface] - if !ok { - hdlrs = []func(string, *Client){} - } - - hdlrs = append(hdlrs, fn) - handlers.extpointHandlers[iface] = hdlrs - - storage.Lock() - for _, p := range storage.plugins { - p.activateWait.L.Lock() - if p.activated() && p.implements(iface) { - p.handlersRun = false - } - p.activateWait.L.Unlock() - } - storage.Unlock() - - handlers.Unlock() -} - -// GetAll returns all the plugins for the specified implementation -func GetAll(imp string) ([]*Plugin, error) { - pluginNames, err := Scan() - if err != nil { - return nil, err - } - - type plLoad struct { - pl *Plugin - err error - } - - chPl := make(chan *plLoad, len(pluginNames)) - var wg sync.WaitGroup - for _, name := range pluginNames { - storage.Lock() - pl, ok := storage.plugins[name] - storage.Unlock() - if ok { - chPl <- &plLoad{pl, nil} - continue - } - - wg.Add(1) - go func(name string) { - defer wg.Done() - pl, err := loadWithRetry(name, false) - chPl <- &plLoad{pl, err} - }(name) - } - - wg.Wait() - close(chPl) - - var out []*Plugin - for pl := range chPl { - if pl.err != nil { - logrus.Error(pl.err) - continue - } - if err := pl.pl.waitActive(); err == nil && pl.pl.implements(imp) { - out = append(out, pl.pl) - } - } - return out, nil -} diff --git a/vendor/github.com/containers/storage/pkg/plugins/plugins_unix.go b/vendor/github.com/containers/storage/pkg/plugins/plugins_unix.go deleted file mode 100644 index 02f1da69..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/plugins_unix.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build !windows - -package plugins - -// BasePath returns the path to which all paths returned by the plugin are relative to. -// For v1 plugins, this always returns the host's root directory. -func (p *Plugin) BasePath() string { - return "/" -} diff --git a/vendor/github.com/containers/storage/pkg/plugins/plugins_windows.go b/vendor/github.com/containers/storage/pkg/plugins/plugins_windows.go deleted file mode 100644 index 3c8d8feb..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/plugins_windows.go +++ /dev/null @@ -1,8 +0,0 @@ -package plugins - -// BasePath returns the path to which all paths returned by the plugin are relative to. -// For Windows v1 plugins, this returns an empty string, since the plugin is already aware -// of the absolute path of the mount. -func (p *Plugin) BasePath() string { - return "" -} diff --git a/vendor/github.com/containers/storage/pkg/plugins/transport/http.go b/vendor/github.com/containers/storage/pkg/plugins/transport/http.go deleted file mode 100644 index 5be146af..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/transport/http.go +++ /dev/null @@ -1,36 +0,0 @@ -package transport - -import ( - "io" - "net/http" -) - -// httpTransport holds an http.RoundTripper -// and information about the scheme and address the transport -// sends request to. -type httpTransport struct { - http.RoundTripper - scheme string - addr string -} - -// NewHTTPTransport creates a new httpTransport. -func NewHTTPTransport(r http.RoundTripper, scheme, addr string) Transport { - return httpTransport{ - RoundTripper: r, - scheme: scheme, - addr: addr, - } -} - -// NewRequest creates a new http.Request and sets the URL -// scheme and address with the transport's fields. -func (t httpTransport) NewRequest(path string, data io.Reader) (*http.Request, error) { - req, err := newHTTPRequest(path, data) - if err != nil { - return nil, err - } - req.URL.Scheme = t.scheme - req.URL.Host = t.addr - return req, nil -} diff --git a/vendor/github.com/containers/storage/pkg/plugins/transport/transport.go b/vendor/github.com/containers/storage/pkg/plugins/transport/transport.go deleted file mode 100644 index d7f1e210..00000000 --- a/vendor/github.com/containers/storage/pkg/plugins/transport/transport.go +++ /dev/null @@ -1,36 +0,0 @@ -package transport - -import ( - "io" - "net/http" - "strings" -) - -// VersionMimetype is the Content-Type the engine sends to plugins. -const VersionMimetype = "application/vnd.docker.plugins.v1.2+json" - -// RequestFactory defines an interface that -// transports can implement to create new requests. -type RequestFactory interface { - NewRequest(path string, data io.Reader) (*http.Request, error) -} - -// Transport defines an interface that plugin transports -// must implement. -type Transport interface { - http.RoundTripper - RequestFactory -} - -// newHTTPRequest creates a new request with a path and a body. -func newHTTPRequest(path string, data io.Reader) (*http.Request, error) { - if !strings.HasPrefix(path, "/") { - path = "/" + path - } - req, err := http.NewRequest("POST", path, data) - if err != nil { - return nil, err - } - req.Header.Add("Accept", VersionMimetype) - return req, nil -} diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 5d04ea1e..add87d3d 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -590,7 +590,13 @@ func (s *store) getGraphDriver() (drivers.Driver, error) { if s.graphDriver != nil { return s.graphDriver, nil } - driver, err := drivers.New(s.graphRoot, s.graphDriverName, s.graphOptions, s.uidMap, s.gidMap) + config := drivers.Options{ + Root: s.graphRoot, + DriverOptions: s.graphOptions, + UIDMaps: s.uidMap, + GIDMaps: s.gidMap, + } + driver, err := drivers.New(s.graphDriverName, config) if err != nil { return nil, err } @@ -769,30 +775,32 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o id = stringid.GenerateRandomID() } - lstore, err := s.LayerStore() - if err != nil { - return nil, err - } - lstores, err := s.ROLayerStores() - if err != nil { - return nil, err - } - var ilayer *Layer - for _, store := range append([]ROLayerStore{lstore}, lstores...) { - store.Lock() - defer store.Unlock() - if modified, err := store.Modified(); modified || err != nil { - store.Load() + if layer != "" { + lstore, err := s.LayerStore() + if err != nil { + return nil, err } - ilayer, err = store.Get(layer) - if err == nil { - break + lstores, err := s.ROLayerStores() + if err != nil { + return nil, err } + var ilayer *Layer + for _, store := range append([]ROLayerStore{lstore}, lstores...) { + store.Lock() + defer store.Unlock() + if modified, err := store.Modified(); modified || err != nil { + store.Load() + } + ilayer, err = store.Get(layer) + if err == nil { + break + } + } + if ilayer == nil { + return nil, ErrLayerUnknown + } + layer = ilayer.ID } - if ilayer == nil { - return nil, ErrLayerUnknown - } - layer = ilayer.ID ristore, err := s.ImageStore() if err != nil { @@ -1168,15 +1176,20 @@ func (s *store) Exists(id string) bool { return false } -func (s *store) SetNames(id string, names []string) error { - deduped := []string{} +func dedupeNames(names []string) []string { seen := make(map[string]bool) + deduped := make([]string, 0, len(names)) for _, name := range names { if _, wasSeen := seen[name]; !wasSeen { seen[name] = true deduped = append(deduped, name) } } + return deduped +} + +func (s *store) SetNames(id string, names []string) error { + deduped := dedupeNames(names) rlstore, err := s.LayerStore() if err != nil { @@ -2243,6 +2256,12 @@ type OptionsConfig struct { // Image stores. Usually used to access Networked File System // for shared image content AdditionalImageStores []string `toml:"additionalimagestores"` + + // Size + Size string `toml:"size"` + + // OverrideKernelCheck + OverrideKernelCheck string `toml:"override_kernel_check"` } // TOML-friendly explicit tables used for conversions. @@ -2286,7 +2305,12 @@ func init() { for _, s := range config.Storage.Options.AdditionalImageStores { DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.imagestore=%s", config.Storage.Driver, s)) } - + if config.Storage.Options.Size != "" { + DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size)) + } + if config.Storage.Options.OverrideKernelCheck != "" { + DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.override_kernel_check=%s", config.Storage.Driver, config.Storage.Options.OverrideKernelCheck)) + } if os.Getenv("STORAGE_DRIVER") != "" { DefaultStoreOptions.GraphDriverName = os.Getenv("STORAGE_DRIVER") } From 51fc13abf3e4937770e9b6a0439ea6a308a98a32 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Tue, 26 Sep 2017 21:29:17 +0000 Subject: [PATCH 2/4] Add documentation on setting up quota Signed-off-by: Daniel J Walsh --- docs/crio.conf.5.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/crio.conf.5.md b/docs/crio.conf.5.md index 01d1bc32..a1700380 100644 --- a/docs/crio.conf.5.md +++ b/docs/crio.conf.5.md @@ -38,9 +38,33 @@ The `crio` table supports the following options: **storage_driver**="" CRIO storage driver (default is "overlay") +Note: + **overlay** and **overlay2** are the same driver + + **storage_option**=[] CRIO storage driver option list (no default) + Values: + + "STORAGE_DRIVER.imagestore=/PATH", + + Paths to additional container image stores. These are read/only and are usually stored on remote network shares, based on overlay storage format. + storage_option=[ "overlay.imagestore=/mnt/overlay", ] + + "STORAGE_DRIVER.size=SIZE" + + Maximum size of a container image. Default is 10GB. The size flag sets quota on the size of container images. + storage_option=[ "overlay.size=1G", ] + +Note: Not all drivers support all options. + +Note: In order to use the **size** option for quota on *overlay* storage you must use the *xfs* file system. The mount point that the *overlay* file system must be setup with the *pquota* flag at mount time. If you are setting up / to be used with quota, you have to modify the linux boot line in /etc/grubq2.conf and add the rootflags=pquota flag. + +Example: + linux16 /vmlinuz-4.12.13-300.fc26.x86_64 root=/dev/mapper/fedora-root ro rd.lvm.lv=fedora/root rd.lvm.lv=fedora/swap rhgb quiet LANG=en_US.UTF-8 rootflags=pquota + + ## CRIO.API TABLE **listen**="" @@ -125,7 +149,7 @@ The `crio` table supports the following options: Path to CNI plugin binaries (default: "/opt/cni/bin/") # SEE ALSO -crio(8), containers-storage.conf(5) +crio(8) # HISTORY Oct 2016, Originally compiled by Aleksa Sarai From 1a41d6ecd47c910a38fe248620a1d8ce329d53a0 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Thu, 28 Sep 2017 11:43:15 -0400 Subject: [PATCH 3/4] Remove references to overlay2 in the code Signed-off-by: Daniel J Walsh --- cmd/kpod/main.go | 2 +- contrib/test/integration/build/cri-o.yml | 10 +++++----- contrib/test/integration/test.yml | 4 ++-- docs/kpod-inspect.1.md | 8 ++++---- docs/kpod-mount.1.md | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/kpod/main.go b/cmd/kpod/main.go index 2936e49f..804f762a 100644 --- a/cmd/kpod/main.go +++ b/cmd/kpod/main.go @@ -98,7 +98,7 @@ func main() { }, cli.StringFlag{ Name: "storage-driver, s", - Usage: "select which storage driver is used to manage storage of images and containers (default is overlay2)", + Usage: "select which storage driver is used to manage storage of images and containers (default is overlay)", }, cli.StringSliceFlag{ Name: "storage-opt", diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml index 8f6cee92..fa025035 100644 --- a/contrib/test/integration/build/cri-o.yml +++ b/contrib/test/integration/build/cri-o.yml @@ -47,10 +47,10 @@ - src: test/redhat_sigstore.yaml dest: /etc/containers/registries.d/registry.access.redhat.com.yaml -- name: run with overlay2 +- name: run with overlay replace: regexp: 'storage_driver = ""' - replace: 'storage_driver = "overlay2"' + replace: 'storage_driver = "overlay"' name: /etc/crio/crio.conf backup: yes @@ -69,11 +69,11 @@ regexp: 'docker\.io' state: present -- name: add overlay2 storage opts on RHEL/CentOS +- name: add overlay storage opts on RHEL/CentOS lineinfile: dest: /etc/crio/crio.conf - line: '"overlay2.override_kernel_check=1"' + line: '"overlay.override_kernel_check=1"' insertafter: 'storage_option = \[' - regexp: 'overlay2\.override_kernel_check=1' + regexp: 'overlay\.override_kernel_check=1' state: present when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml index b079cd02..5c58b4c6 100644 --- a/contrib/test/integration/test.yml +++ b/contrib/test/integration/test.yml @@ -17,7 +17,7 @@ - name: set extra storage options set_fact: - extra_storage_opts: " --storage-opt overlay2.override_kernel_check=1" + extra_storage_opts: " --storage-opt overlay.override_kernel_check=1" when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' - name: set extra shell for non-xunit tests @@ -26,7 +26,7 @@ when: not xunit - name: run integration tests - shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay2{{ extra_storage_opts | default('') }}' make localintegration{{ extra_shell_suffix }}" + shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration{{ extra_shell_suffix }}" args: chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" ignore_errors: yes diff --git a/docs/kpod-inspect.1.md b/docs/kpod-inspect.1.md index 21d41a15..633f1fa4 100644 --- a/docs/kpod-inspect.1.md +++ b/docs/kpod-inspect.1.md @@ -146,11 +146,11 @@ kpod inspect redis:alpine "Size": 3965955, "VirtualSize": 19808086, "GraphDriver": { - "Name": "overlay2", + "Name": "overlay", "Data": { - "MergedDir": "/var/lib/containers/storage/overlay2/2059d805c90e034cb773d9722232ef018a72143dd31113b470fb876baeccd700/merged", - "UpperDir": "/var/lib/containers/storage/overlay2/2059d805c90e034cb773d9722232ef018a72143dd31113b470fb876baeccd700/diff", - "WorkDir": "/var/lib/containers/storage/overlay2/2059d805c90e034cb773d9722232ef018a72143dd31113b470fb876baeccd700/work" + "MergedDir": "/var/lib/containers/storage/overlay/2059d805c90e034cb773d9722232ef018a72143dd31113b470fb876baeccd700/merged", + "UpperDir": "/var/lib/containers/storage/overlay/2059d805c90e034cb773d9722232ef018a72143dd31113b470fb876baeccd700/diff", + "WorkDir": "/var/lib/containers/storage/overlay/2059d805c90e034cb773d9722232ef018a72143dd31113b470fb876baeccd700/work" } }, "RootFS": { diff --git a/docs/kpod-mount.1.md b/docs/kpod-mount.1.md index 599edce6..25ccd937 100644 --- a/docs/kpod-mount.1.md +++ b/docs/kpod-mount.1.md @@ -38,13 +38,13 @@ SELinux label for the mount point kpod mount c831414b10a3 -/var/lib/containers/storage/overlay2/f3ac502d97b5681989dff84dfedc8354239bcecbdc2692f9a639f4e080a02364/merged +/var/lib/containers/storage/overlay/f3ac502d97b5681989dff84dfedc8354239bcecbdc2692f9a639f4e080a02364/merged kpod mount -c831414b10a3 /var/lib/containers/storage/overlay2/f3ac502d97b5681989dff84dfedc8354239bcecbdc2692f9a639f4e080a02364/merged +c831414b10a3 /var/lib/containers/storage/overlay/f3ac502d97b5681989dff84dfedc8354239bcecbdc2692f9a639f4e080a02364/merged -a7060253093b /var/lib/containers/storage/overlay2/0ff7d7ca68bed1ace424f9df154d2dd7b5a125c19d887f17653cbcd5b6e30ba1/merged +a7060253093b /var/lib/containers/storage/overlay/0ff7d7ca68bed1ace424f9df154d2dd7b5a125c19d887f17653cbcd5b6e30ba1/merged ## SEE ALSO kpod(1), kpod-umount(1), mount(8) From 6866589be7baa243a10c884108eb597913dd9164 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Fri, 29 Sep 2017 05:16:49 -0400 Subject: [PATCH 4/4] Drop btrfs support from RHEL and CentOS Packages are no longer available to build on RHEL and CentOS and btrfs is not longer supported, so we should not build with it. Signed-off-by: Daniel J Walsh Signed-off-by: Daniel J Walsh --- contrib/test/integration/system.yml | 8 ++++++++ contrib/test/integration/vars.yml | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml index 97695b7b..5141e433 100644 --- a/contrib/test/integration/system.yml +++ b/contrib/test/integration/system.yml @@ -19,6 +19,14 @@ async: 600 poll: 10 +- name: Add Btrfs for Fedora + package: + name: "{{ item }}" + state: present + with_items: + - btrfs-progs-devel + when: ansible_distribution in ['Fedora'] + - name: Update all packages package: name: '*' diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml index 552fa7e3..571e6bd6 100644 --- a/contrib/test/integration/vars.yml +++ b/contrib/test/integration/vars.yml @@ -4,7 +4,6 @@ xunit: false rpm_pkgs: - - btrfs-progs-devel - container-selinux - curl - device-mapper-devel