Bump image, storage, and image-spec

Bump containers/image (pulling in its new dependency on ostree-go),
containers/storage, and updated image-spec.

This pulls in the OCI v1.0 specifications and code that allows us to
support 1.0 images.

Signed-off-by: Dan Walsh <dwalsh@redhat.com>
Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
This commit is contained in:
Dan Walsh 2017-07-20 16:31:51 -04:00 committed by Nalin Dahyabhai
parent 5138691c3b
commit d76645680f
117 changed files with 3965 additions and 991 deletions

View file

@ -1,6 +1,6 @@
`storage` is a Go library which aims to provide methods for storing filesystem
layers, container images, and containers. An `oci-storage` CLI wrapper is also
included for manual and scripting use.
layers, container images, and containers. A `containers-storage` CLI wrapper
is also included for manual and scripting use.
To build the CLI wrapper, use 'make build-binary'.

View file

@ -50,6 +50,12 @@ type Container struct {
// that has been stored, if they're known.
BigDataSizes map[string]int64 `json:"big-data-sizes,omitempty"`
// Created is the datestamp for when this container was created. Older
// versions of the library did not track this information, so callers
// will likely want to use the IsZero() method to verify that a value
// is set before using it.
Created time.Time `json:"created,omitempty"`
Flags map[string]interface{} `json:"flags,omitempty"`
}
@ -93,7 +99,7 @@ type ContainerStore interface {
type containerStore struct {
lockfile Locker
dir string
containers []Container
containers []*Container
idindex *truncindex.TruncIndex
byid map[string]*Container
bylayer map[string]*Container
@ -101,7 +107,11 @@ type containerStore struct {
}
func (r *containerStore) Containers() ([]Container, error) {
return r.containers, nil
containers := make([]Container, len(r.containers))
for i := range r.containers {
containers[i] = *(r.containers[i])
}
return containers, nil
}
func (r *containerStore) containerspath() string {
@ -123,7 +133,7 @@ func (r *containerStore) Load() error {
if err != nil && !os.IsNotExist(err) {
return err
}
containers := []Container{}
containers := []*Container{}
layers := make(map[string]*Container)
idlist := []string{}
ids := make(map[string]*Container)
@ -131,14 +141,14 @@ func (r *containerStore) Load() error {
if err = json.Unmarshal(data, &containers); len(data) == 0 || err == nil {
for n, container := range containers {
idlist = append(idlist, container.ID)
ids[container.ID] = &containers[n]
layers[container.LayerID] = &containers[n]
ids[container.ID] = containers[n]
layers[container.LayerID] = containers[n]
for _, name := range container.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
needSave = true
}
names[name] = &containers[n]
names[name] = containers[n]
}
}
}
@ -148,7 +158,6 @@ func (r *containerStore) Load() error {
r.bylayer = layers
r.byname = names
if needSave {
r.Touch()
return r.Save()
}
return nil
@ -163,6 +172,7 @@ func (r *containerStore) Save() error {
if err != nil {
return err
}
defer r.Touch()
return ioutils.AtomicWriteFile(rpath, jdata, 0600)
}
@ -179,7 +189,7 @@ func newContainerStore(dir string) (ContainerStore, error) {
cstore := containerStore{
lockfile: lockfile,
dir: dir,
containers: []Container{},
containers: []*Container{},
byid: make(map[string]*Container),
bylayer: make(map[string]*Container),
byname: make(map[string]*Container),
@ -241,7 +251,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
}
}
if err == nil {
newContainer := Container{
container = &Container{
ID: id,
Names: names,
ImageID: image,
@ -249,10 +259,10 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
Metadata: metadata,
BigDataNames: []string{},
BigDataSizes: make(map[string]int64),
Created: time.Now().UTC(),
Flags: make(map[string]interface{}),
}
r.containers = append(r.containers, newContainer)
container = &r.containers[len(r.containers)-1]
r.containers = append(r.containers, container)
r.byid[id] = container
r.idindex.Add(id)
r.bylayer[layer] = container
@ -306,10 +316,11 @@ func (r *containerStore) Delete(id string) error {
return ErrContainerUnknown
}
id = container.ID
newContainers := []Container{}
for _, candidate := range r.containers {
if candidate.ID != id {
newContainers = append(newContainers, candidate)
toDeleteIndex := -1
for i, candidate := range r.containers {
if candidate.ID == id {
toDeleteIndex = i
break
}
}
delete(r.byid, id)
@ -318,7 +329,14 @@ func (r *containerStore) Delete(id string) error {
for _, name := range container.Names {
delete(r.byname, name)
}
r.containers = newContainers
if toDeleteIndex != -1 {
// delete the container at toDeleteIndex
if toDeleteIndex == len(r.containers)-1 {
r.containers = r.containers[:len(r.containers)-1]
} else {
r.containers = append(r.containers[:toDeleteIndex], r.containers[toDeleteIndex+1:]...)
}
}
if err := r.Save(); err != nil {
return err
}
@ -437,6 +455,10 @@ func (r *containerStore) Modified() (bool, error) {
return r.lockfile.Modified()
}
func (r *containerStore) IsReadWrite() bool {
return r.lockfile.IsReadWrite()
}
func (r *containerStore) TouchedSince(when time.Time) bool {
return r.lockfile.TouchedSince(when)
}

View file

@ -47,6 +47,7 @@ import (
rsystem "github.com/opencontainers/runc/libcontainer/system"
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
)
var (
@ -81,7 +82,7 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
// Try to load the aufs kernel module
if err := supportsAufs(); err != nil {
return nil, graphdriver.ErrNotSupported
return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel does not support aufs")
}
fsMagic, err := graphdriver.GetFSMagic(root)
@ -95,7 +96,7 @@ func Init(root string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
switch fsMagic {
case graphdriver.FsMagicAufs, graphdriver.FsMagicBtrfs, graphdriver.FsMagicEcryptfs:
logrus.Errorf("AUFS is not supported over %s", backingFs)
return nil, graphdriver.ErrIncompatibleFS
return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "AUFS is not supported over %q", backingFs)
}
paths := []string{
@ -372,6 +373,12 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) {
})
}
// AdditionalImageStores returns additional image stores supported by the driver
func (a *Driver) AdditionalImageStores() []string {
var imageStores []string
return imageStores
}
type fileGetNilCloser struct {
storage.FileGetter
}

View file

@ -2,7 +2,7 @@
package aufs
import "errors"
import "github.com/pkg/errors"
// MsRemount declared to specify a non-linux system mount.
const MsRemount = 0

View file

@ -29,6 +29,7 @@ import (
"github.com/containers/storage/pkg/parsers"
"github.com/docker/go-units"
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
)
func init() {
@ -55,7 +56,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
}
if fsMagic != graphdriver.FsMagicBtrfs {
return nil, graphdriver.ErrPrerequisites
return nil, errors.Wrapf(graphdriver.ErrPrerequisites, "%q is not on a btrfs filesystem", home)
}
rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
@ -518,3 +519,9 @@ func (d *Driver) Exists(id string) bool {
_, err := os.Stat(dir)
return err == nil
}
// AdditionalImageStores returns additional image stores supported by the driver
func (d *Driver) AdditionalImageStores() []string {
var imageStores []string
return imageStores
}

View file

@ -5,7 +5,6 @@ package devmapper
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -28,10 +27,10 @@ import (
"github.com/containers/storage/pkg/loopback"
"github.com/containers/storage/pkg/mount"
"github.com/containers/storage/pkg/parsers"
"github.com/containers/storage/storageversion"
"github.com/docker/go-units"
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
)
var (
@ -1475,7 +1474,7 @@ func determineDriverCapabilities(version string) error {
versionSplit := strings.Split(version, ".")
major, err := strconv.Atoi(versionSplit[0])
if err != nil {
return graphdriver.ErrNotSupported
return errors.Wrapf(graphdriver.ErrNotSupported, "unable to parse driver major version %q as a number", versionSplit[0])
}
if major > 4 {
@ -1489,7 +1488,7 @@ func determineDriverCapabilities(version string) error {
minor, err := strconv.Atoi(versionSplit[1])
if err != nil {
return graphdriver.ErrNotSupported
return errors.Wrapf(graphdriver.ErrNotSupported, "unable to parse driver minor version %q as a number", versionSplit[1])
}
/*
@ -1656,11 +1655,11 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
version, err := devicemapper.GetDriverVersion()
if err != nil {
// Can't even get driver version, assume not supported
return graphdriver.ErrNotSupported
return errors.Wrap(graphdriver.ErrNotSupported, "unable to determine version of device mapper")
}
if err := determineDriverCapabilities(version); err != nil {
return graphdriver.ErrNotSupported
return errors.Wrap(graphdriver.ErrNotSupported, "unable to determine device mapper driver capabilities")
}
if err := devices.enableDeferredRemovalDeletion(); err != nil {
@ -1668,17 +1667,17 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
}
// https://github.com/docker/docker/issues/4036
if supported := devicemapper.UdevSetSyncSupport(true); !supported {
if storageversion.IAmStatic == "true" {
logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option")
} else {
logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option")
}
if !devices.overrideUdevSyncCheck {
return graphdriver.ErrNotSupported
}
}
// if supported := devicemapper.UdevSetSyncSupport(true); !supported {
// if storageversion.IAmStatic == "true" {
// logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option")
// } else {
// logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option")
// }
//
// if !devices.overrideUdevSyncCheck {
// return graphdriver.ErrNotSupported
// }
// }
//create the root dir of the devmapper driver ownership to match this
//daemon's remapped root uid/gid so containers can start properly
@ -1734,6 +1733,15 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error {
metadataFile *os.File
)
fsMagic, err := graphdriver.GetFSMagic(devices.loopbackDir())
if err != nil {
return err
}
switch fsMagic {
case graphdriver.FsMagicAufs:
return errors.Errorf("devmapper: Loopback devices can not be created on AUFS filesystems")
}
if devices.dataDevice == "" {
// Make sure the sparse images exist in <root>/devicemapper/data
@ -1960,7 +1968,7 @@ func (devices *DeviceSet) deleteTransaction(info *devInfo, syncDelete bool) erro
// If syncDelete is true, we want to return error. If deferred
// deletion is not enabled, we return an error. If error is
// something other then EBUSY, return an error.
if syncDelete || !devices.deferredDelete || err != devicemapper.ErrBusy {
if syncDelete || !devices.deferredDelete || errors.Cause(err) != devicemapper.ErrBusy {
logrus.Debugf("devmapper: Error deleting device: %s", err)
return err
}
@ -2115,7 +2123,7 @@ func (devices *DeviceSet) removeDevice(devname string) error {
if err == nil {
break
}
if err != devicemapper.ErrBusy {
if errors.Cause(err) != devicemapper.ErrBusy {
return err
}
@ -2150,12 +2158,12 @@ func (devices *DeviceSet) cancelDeferredRemoval(info *devInfo) error {
break
}
if err == devicemapper.ErrEnxio {
if errors.Cause(err) == devicemapper.ErrEnxio {
// Device is probably already gone. Return success.
return nil
}
if err != devicemapper.ErrBusy {
if errors.Cause(err) != devicemapper.ErrBusy {
return err
}

View file

@ -224,3 +224,9 @@ func (d *Driver) Put(id string) error {
func (d *Driver) Exists(id string) bool {
return d.DeviceSet.HasDevice(id)
}
// AdditionalImageStores returns additional image stores supported by the driver
func (d *Driver) AdditionalImageStores() []string {
var imageStores []string
return imageStores
}

View file

@ -1,13 +1,13 @@
package graphdriver
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/Sirupsen/logrus"
"github.com/pkg/errors"
"github.com/vbatts/tar-split/tar/storage"
"github.com/containers/storage/pkg/archive"
@ -74,6 +74,8 @@ type ProtoDriver interface {
// held by the driver, e.g., unmounting all layered filesystems
// known to this driver.
Cleanup() error
// AdditionalImageStores returns additional image stores supported by the driver
AdditionalImageStores() []string
}
// Driver is the interface for layered/snapshot file system drivers.
@ -142,7 +144,7 @@ func GetDriver(name, home string, options []string, uidMaps, gidMaps []idtools.I
return pluginDriver, nil
}
logrus.Errorf("Failed to GetDriver graph %s %s", name, home)
return nil, ErrNotSupported
return nil, errors.Wrapf(ErrNotSupported, "failed to GetDriver graph %s %s", name, home)
}
// getBuiltinDriver initializes and returns the registered driver, but does not try to load from plugins
@ -151,7 +153,7 @@ func getBuiltinDriver(name, home string, options []string, uidMaps, gidMaps []id
return initFunc(filepath.Join(home, name), options, uidMaps, gidMaps)
}
logrus.Errorf("Failed to built-in GetDriver graph %s %s", name, home)
return nil, ErrNotSupported
return nil, errors.Wrapf(ErrNotSupported, "failed to built-in GetDriver graph %s %s", name, home)
}
// New creates the driver and initializes it at the specified root.
@ -226,7 +228,8 @@ func New(root string, name string, options []string, uidMaps, gidMaps []idtools.
// isDriverNotSupported returns true if the error initializing
// the graph driver is a non-supported error.
func isDriverNotSupported(err error) bool {
return err == ErrNotSupported || err == ErrPrerequisites || err == ErrIncompatibleFS
cause := errors.Cause(err)
return cause == ErrNotSupported || cause == ErrPrerequisites || cause == ErrIncompatibleFS
}
// scanPriorDrivers returns an un-ordered scan of directories of prior storage drivers

View file

@ -53,7 +53,7 @@ const (
var (
// Slice of drivers that should be used in an order
priority = []string{
"overlay2",
"overlay",
"devicemapper",
"aufs",
"btrfs",

View file

@ -20,6 +20,7 @@ import (
"unsafe"
log "github.com/Sirupsen/logrus"
"github.com/pkg/errors"
)
const (
@ -56,7 +57,7 @@ func Mounted(fsType FsMagic, mountPath string) (bool, error) {
(buf.f_basetype[3] != 0) {
log.Debugf("[zfs] no zfs dataset found for rootdir '%s'", mountPath)
C.free(unsafe.Pointer(buf))
return false, ErrPrerequisites
return false, errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", mountPath)
}
C.free(unsafe.Pointer(buf))

View file

@ -4,12 +4,12 @@ package overlay
import (
"bufio"
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strconv"
"strings"
"syscall"
@ -26,6 +26,7 @@ import (
"github.com/containers/storage/pkg/parsers/kernel"
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
)
var (
@ -82,6 +83,7 @@ type Driver struct {
uidMaps []idtools.IDMap
gidMaps []idtools.IDMap
ctr *graphdriver.RefCounter
opts *overlayOptions
}
var backingFs = "<unknown>"
@ -100,7 +102,7 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool
}
if err := supportsOverlay(); err != nil {
return nil, graphdriver.ErrNotSupported
return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel does not support overlay fs")
}
// require kernel 4.0.0 to ensure multiple lower dirs are supported
@ -110,7 +112,7 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool
}
if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 0, Minor: 0}) < 0 {
if !opts.overrideKernelCheck {
return nil, graphdriver.ErrNotSupported
return nil, errors.Wrap(graphdriver.ErrNotSupported, "kernel too old to provide multiple lowers feature for overlay")
}
logrus.Warnf("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update")
}
@ -127,7 +129,7 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool
switch fsMagic {
case graphdriver.FsMagicBtrfs, graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs:
logrus.Errorf("'overlay' is not supported over %s", backingFs)
return nil, graphdriver.ErrIncompatibleFS
return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s", backingFs)
}
rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
@ -149,6 +151,7 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool
uidMaps: uidMaps,
gidMaps: gidMaps,
ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)),
opts: opts,
}
return d, nil
@ -170,6 +173,7 @@ func InitAsOverlay2(home string, options []string, uidMaps, gidMaps []idtools.ID
type overlayOptions struct {
overrideKernelCheck bool
imageStores []string
}
func parseOptions(options []string) (*overlayOptions, error) {
@ -186,6 +190,22 @@ func parseOptions(options []string) (*overlayOptions, error) {
if err != nil {
return nil, err
}
case "overlay.imagestore":
// Additional read only image stores to use for lower paths
for _, store := range strings.Split(val, ",") {
store = filepath.Clean(store)
if !filepath.IsAbs(store) {
return nil, fmt.Errorf("overlay: image path %q is not absolute. Can not be relative", store)
}
st, err := os.Stat(store)
if err != nil {
return nil, fmt.Errorf("overlay: Can't stat imageStore dir %s: %v", store, err)
}
if !st.IsDir() {
return nil, fmt.Errorf("overlay: image path %q must be a directory", store)
}
o.imageStores = append(o.imageStores, store)
}
default:
return nil, fmt.Errorf("overlay: Unknown option %s", key)
}
@ -211,7 +231,7 @@ func supportsOverlay() error {
}
}
logrus.Error("'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
return graphdriver.ErrNotSupported
return errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.")
}
func (d *Driver) String() string {
@ -357,8 +377,18 @@ func (d *Driver) getLower(parent string) (string, error) {
return strings.Join(lowers, ":"), nil
}
func (d *Driver) dir(id string) string {
return path.Join(d.home, id)
func (d *Driver) dir(val string) string {
newpath := path.Join(d.home, val)
if _, err := os.Stat(newpath); err != nil {
for _, p := range d.AdditionalImageStores() {
l := path.Join(p, d.name, val)
_, err = os.Stat(l)
if err == nil {
return l
}
}
}
return newpath
}
func (d *Driver) getLowerDirs(id string) ([]string, error) {
@ -366,11 +396,12 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
lowers, err := ioutil.ReadFile(path.Join(d.dir(id), lowerFile))
if err == nil {
for _, s := range strings.Split(string(lowers), ":") {
lp, err := os.Readlink(path.Join(d.home, s))
lower := d.dir(s)
lp, err := os.Readlink(lower)
if err != nil {
return nil, err
}
lowersArray = append(lowersArray, path.Clean(path.Join(d.home, "link", lp)))
lowersArray = append(lowersArray, path.Clean(d.dir(path.Join("link", lp))))
}
} else if !os.IsNotExist(err) {
return nil, err
@ -411,6 +442,31 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
return "", err
}
newlowers := ""
for _, l := range strings.Split(string(lowers), ":") {
lower := ""
newpath := path.Join(d.home, l)
if _, err := os.Stat(newpath); err != nil {
for _, p := range d.AdditionalImageStores() {
lower = path.Join(p, d.name, l)
if _, err2 := os.Stat(lower); err2 == nil {
break
}
lower = ""
}
if lower == "" {
return "", fmt.Errorf("Can't stat lower layer %q: %v", newpath, err)
}
} else {
lower = l
}
if newlowers == "" {
newlowers = lower
} else {
newlowers = newlowers + ":" + lower
}
}
mergedDir := path.Join(dir, "merged")
if count := d.ctr.Increment(mergedDir); count > 1 {
return mergedDir, nil
@ -424,7 +480,7 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
}()
workDir := path.Join(dir, "work")
opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", string(lowers), path.Join(id, "diff"), path.Join(id, "work"))
opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", newlowers, path.Join(id, "diff"), path.Join(id, "work"))
mountLabel = label.FormatMountLabel(opts, mountLabel)
if len(mountLabel) > syscall.Getpagesize() {
return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountLabel))
@ -527,3 +583,8 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) {
return archive.OverlayChanges(layers, diffPath)
}
// AdditionalImageStores returns additional image stores supported by the driver
func (d *Driver) AdditionalImageStores() []string {
return d.opts.imageStores
}

View file

@ -3,10 +3,10 @@
package graphdriver
import (
"errors"
"fmt"
"github.com/containers/storage/pkg/archive"
"github.com/pkg/errors"
)
type graphDriverProxy struct {

View file

@ -143,3 +143,9 @@ func (d *Driver) Exists(id string) bool {
_, err := os.Stat(d.dir(id))
return err == nil
}
// AdditionalImageStores returns additional image stores supported by the driver
func (d *Driver) AdditionalImageStores() []string {
var imageStores []string
return imageStores
}

View file

@ -20,6 +20,7 @@ import (
"github.com/containers/storage/pkg/parsers"
zfs "github.com/mistifyio/go-zfs"
"github.com/opencontainers/selinux/go-selinux/label"
"github.com/pkg/errors"
)
type zfsOptions struct {
@ -47,13 +48,13 @@ func Init(base string, opt []string, uidMaps, gidMaps []idtools.IDMap) (graphdri
if _, err := exec.LookPath("zfs"); err != nil {
logrus.Debugf("[zfs] zfs command is not available: %v", err)
return nil, graphdriver.ErrPrerequisites
return nil, errors.Wrap(graphdriver.ErrPrerequisites, "the 'zfs' command is not available")
}
file, err := os.OpenFile("/dev/zfs", os.O_RDWR, 600)
if err != nil {
logrus.Debugf("[zfs] cannot open /dev/zfs: %v", err)
return nil, graphdriver.ErrPrerequisites
return nil, errors.Wrapf(graphdriver.ErrPrerequisites, "could not open /dev/zfs: %v", err)
}
defer file.Close()
@ -403,3 +404,9 @@ func (d *Driver) Exists(id string) bool {
defer d.Unlock()
return d.filesystemsCache[d.zfsPath(id)] == true
}
// AdditionalImageStores returns additional image stores supported by the driver
func (d *Driver) AdditionalImageStores() []string {
var imageStores []string
return imageStores
}

View file

@ -7,6 +7,7 @@ import (
"github.com/Sirupsen/logrus"
"github.com/containers/storage/drivers"
"github.com/pkg/errors"
)
func checkRootdirFs(rootdir string) error {
@ -18,7 +19,7 @@ func checkRootdirFs(rootdir string) error {
// on FreeBSD buf.Fstypename contains ['z', 'f', 's', 0 ... ]
if (buf.Fstypename[0] != 122) || (buf.Fstypename[1] != 102) || (buf.Fstypename[2] != 115) || (buf.Fstypename[3] != 0) {
logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir)
return graphdriver.ErrPrerequisites
return errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", rootdir)
}
return nil

View file

@ -6,6 +6,7 @@ import (
"github.com/Sirupsen/logrus"
"github.com/containers/storage/drivers"
"github.com/pkg/errors"
)
func checkRootdirFs(rootdir string) error {
@ -16,7 +17,7 @@ func checkRootdirFs(rootdir string) error {
if graphdriver.FsMagic(buf.Type) != graphdriver.FsMagicZfs {
logrus.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir)
return graphdriver.ErrPrerequisites
return errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", rootdir)
}
return nil

View file

@ -22,6 +22,7 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/containers/storage/drivers"
"github.com/pkg/errors"
)
func checkRootdirFs(rootdir string) error {
@ -34,7 +35,7 @@ func checkRootdirFs(rootdir string) error {
(buf.f_basetype[3] != 0) {
log.Debugf("[zfs] no zfs dataset found for rootdir '%s'", rootdir)
C.free(unsafe.Pointer(buf))
return graphdriver.ErrPrerequisites
return errors.Wrapf(graphdriver.ErrPrerequisites, "no zfs dataset found for rootdir '%s'", rootdir)
}
C.free(unsafe.Pointer(buf))

View file

@ -2,7 +2,6 @@ package storage
import (
"encoding/json"
"errors"
"io/ioutil"
"os"
"path/filepath"
@ -11,6 +10,7 @@ import (
"github.com/containers/storage/pkg/ioutils"
"github.com/containers/storage/pkg/stringid"
"github.com/containers/storage/pkg/truncindex"
"github.com/pkg/errors"
)
var (
@ -46,24 +46,20 @@ type Image struct {
// that has been stored, if they're known.
BigDataSizes map[string]int64 `json:"big-data-sizes,omitempty"`
// Created is the datestamp for when this image was created. Older
// versions of the library did not track this information, so callers
// will likely want to use the IsZero() method to verify that a value
// is set before using it.
Created time.Time `json:"created,omitempty"`
Flags map[string]interface{} `json:"flags,omitempty"`
}
// ImageStore provides bookkeeping for information about Images.
type ImageStore interface {
FileBasedStore
MetadataStore
BigDataStore
FlaggableStore
// Create creates an image that has a specified ID (or a random one) and
// optional names, using the specified layer as its topmost (hopefully
// read-only) layer. That layer can be referenced by multiple images.
Create(id string, names []string, layer, metadata string) (*Image, error)
// SetNames replaces the list of names associated with an image with the
// supplied values.
SetNames(id string, names []string) error
// ROImageStore provides bookkeeping for information about Images.
type ROImageStore interface {
ROFileBasedStore
ROMetadataStore
ROBigDataStore
// Exists checks if there is an image with the given ID or name.
Exists(id string) bool
@ -71,12 +67,6 @@ type ImageStore interface {
// Get retrieves information about an image given an ID or name.
Get(id string) (*Image, error)
// Delete removes the record of the image.
Delete(id string) error
// Wipe removes records of all images.
Wipe() error
// Lookup attempts to translate a name to an ID. Most methods do this
// implicitly.
Lookup(name string) (string, error)
@ -85,17 +75,45 @@ type ImageStore interface {
Images() ([]Image, error)
}
// ImageStore provides bookkeeping for information about Images.
type ImageStore interface {
ROImageStore
RWFileBasedStore
RWMetadataStore
RWBigDataStore
FlaggableStore
// Create creates an image that has a specified ID (or a random one) and
// optional names, using the specified layer as its topmost (hopefully
// read-only) layer. That layer can be referenced by multiple images.
Create(id string, names []string, layer, metadata string, created time.Time) (*Image, error)
// SetNames replaces the list of names associated with an image with the
// supplied values.
SetNames(id string, names []string) error
// Delete removes the record of the image.
Delete(id string) error
// Wipe removes records of all images.
Wipe() error
}
type imageStore struct {
lockfile Locker
dir string
images []Image
images []*Image
idindex *truncindex.TruncIndex
byid map[string]*Image
byname map[string]*Image
}
func (r *imageStore) Images() ([]Image, error) {
return r.images, nil
images := make([]Image, len(r.images))
for i := range r.images {
images[i] = *(r.images[i])
}
return images, nil
}
func (r *imageStore) imagespath() string {
@ -111,41 +129,46 @@ func (r *imageStore) datapath(id, key string) string {
}
func (r *imageStore) Load() error {
needSave := false
shouldSave := false
rpath := r.imagespath()
data, err := ioutil.ReadFile(rpath)
if err != nil && !os.IsNotExist(err) {
return err
}
images := []Image{}
images := []*Image{}
idlist := []string{}
ids := make(map[string]*Image)
names := make(map[string]*Image)
if err = json.Unmarshal(data, &images); len(data) == 0 || err == nil {
for n, image := range images {
ids[image.ID] = &images[n]
ids[image.ID] = images[n]
idlist = append(idlist, image.ID)
for _, name := range image.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
needSave = true
shouldSave = true
}
names[name] = &images[n]
names[name] = images[n]
}
}
}
if shouldSave && !r.IsReadWrite() {
return errors.New("image store assigns the same name to multiple images")
}
r.images = images
r.idindex = truncindex.NewTruncIndex(idlist)
r.byid = ids
r.byname = names
if needSave {
r.Touch()
if shouldSave {
return r.Save()
}
return nil
}
func (r *imageStore) Save() error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify the image store at %q", r.imagespath())
}
rpath := r.imagespath()
if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
return err
@ -154,6 +177,7 @@ func (r *imageStore) Save() error {
if err != nil {
return err
}
defer r.Touch()
return ioutils.AtomicWriteFile(rpath, jdata, 0600)
}
@ -170,7 +194,27 @@ func newImageStore(dir string) (ImageStore, error) {
istore := imageStore{
lockfile: lockfile,
dir: dir,
images: []Image{},
images: []*Image{},
byid: make(map[string]*Image),
byname: make(map[string]*Image),
}
if err := istore.Load(); err != nil {
return nil, err
}
return &istore, nil
}
func newROImageStore(dir string) (ROImageStore, error) {
lockfile, err := GetROLockfile(filepath.Join(dir, "images.lock"))
if err != nil {
return nil, err
}
lockfile.Lock()
defer lockfile.Unlock()
istore := imageStore{
lockfile: lockfile,
dir: dir,
images: []*Image{},
byid: make(map[string]*Image),
byname: make(map[string]*Image),
}
@ -193,6 +237,9 @@ func (r *imageStore) lookup(id string) (*Image, bool) {
}
func (r *imageStore) ClearFlag(id string, flag string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to clear flags on images at %q", r.imagespath())
}
image, ok := r.lookup(id)
if !ok {
return ErrImageUnknown
@ -202,6 +249,9 @@ func (r *imageStore) ClearFlag(id string, flag string) error {
}
func (r *imageStore) SetFlag(id string, flag string, value interface{}) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to set flags on images at %q", r.imagespath())
}
image, ok := r.lookup(id)
if !ok {
return ErrImageUnknown
@ -210,7 +260,10 @@ func (r *imageStore) SetFlag(id string, flag string, value interface{}) error {
return r.Save()
}
func (r *imageStore) Create(id string, names []string, layer, metadata string) (image *Image, err error) {
func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time) (image *Image, err error) {
if !r.IsReadWrite() {
return nil, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new images at %q", r.imagespath())
}
if id == "" {
id = stringid.GenerateRandomID()
_, idInUse := r.byid[id]
@ -227,18 +280,21 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string) (
return nil, ErrDuplicateName
}
}
if created.IsZero() {
created = time.Now().UTC()
}
if err == nil {
newImage := Image{
image = &Image{
ID: id,
Names: names,
TopLayer: layer,
Metadata: metadata,
BigDataNames: []string{},
BigDataSizes: make(map[string]int64),
Created: created,
Flags: make(map[string]interface{}),
}
r.images = append(r.images, newImage)
image = &r.images[len(r.images)-1]
r.images = append(r.images, image)
r.idindex.Add(id)
r.byid[id] = image
for _, name := range names {
@ -257,6 +313,9 @@ func (r *imageStore) Metadata(id string) (string, error) {
}
func (r *imageStore) SetMetadata(id, metadata string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify image metadata at %q", r.imagespath())
}
if image, ok := r.lookup(id); ok {
image.Metadata = metadata
return r.Save()
@ -269,6 +328,9 @@ func (r *imageStore) removeName(image *Image, name string) {
}
func (r *imageStore) SetNames(id string, names []string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to change image name assignments at %q", r.imagespath())
}
if image, ok := r.lookup(id); ok {
for _, name := range image.Names {
delete(r.byname, name)
@ -286,15 +348,18 @@ func (r *imageStore) SetNames(id string, names []string) error {
}
func (r *imageStore) Delete(id string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to delete images at %q", r.imagespath())
}
image, ok := r.lookup(id)
if !ok {
return ErrImageUnknown
}
id = image.ID
newImages := []Image{}
for _, candidate := range r.images {
if candidate.ID != id {
newImages = append(newImages, candidate)
toDeleteIndex := -1
for i, candidate := range r.images {
if candidate.ID == id {
toDeleteIndex = i
}
}
delete(r.byid, id)
@ -302,7 +367,14 @@ func (r *imageStore) Delete(id string) error {
for _, name := range image.Names {
delete(r.byname, name)
}
r.images = newImages
if toDeleteIndex != -1 {
// delete the image at toDeleteIndex
if toDeleteIndex == len(r.images)-1 {
r.images = r.images[:len(r.images)-1]
} else {
r.images = append(r.images[:toDeleteIndex], r.images[toDeleteIndex+1:]...)
}
}
if err := r.Save(); err != nil {
return err
}
@ -359,6 +431,9 @@ func (r *imageStore) BigDataNames(id string) ([]string, error) {
}
func (r *imageStore) SetBigData(id, key string, data []byte) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to save data items associated with images at %q", r.imagespath())
}
image, ok := r.lookup(id)
if !ok {
return ErrImageUnknown
@ -393,6 +468,9 @@ func (r *imageStore) SetBigData(id, key string, data []byte) error {
}
func (r *imageStore) Wipe() error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to delete images at %q", r.imagespath())
}
ids := []string{}
for id := range r.byid {
ids = append(ids, id)
@ -421,6 +499,10 @@ func (r *imageStore) Modified() (bool, error) {
return r.lockfile.Modified()
}
func (r *imageStore) IsReadWrite() bool {
return r.lockfile.IsReadWrite()
}
func (r *imageStore) TouchedSince(when time.Time) bool {
return r.lockfile.TouchedSince(when)
}

View file

@ -4,7 +4,6 @@ import (
"bytes"
"compress/gzip"
"encoding/json"
"errors"
"io"
"io/ioutil"
"os"
@ -16,6 +15,8 @@ import (
"github.com/containers/storage/pkg/ioutils"
"github.com/containers/storage/pkg/stringid"
"github.com/containers/storage/pkg/truncindex"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/vbatts/tar-split/tar/asm"
"github.com/vbatts/tar-split/tar/storage"
)
@ -66,6 +67,38 @@ type Layer struct {
// mounted at the mount point.
MountCount int `json:"-"`
// Created is the datestamp for when this layer was created. Older
// versions of the library did not track this information, so callers
// will likely want to use the IsZero() method to verify that a value
// is set before using it.
Created time.Time `json:"created,omitempty"`
// CompressedDigest is the digest of the blob that was last passed to
// ApplyDiff() or Put(), as it was presented to us.
CompressedDigest digest.Digest `json:"compressed-diff-digest,omitempty"`
// CompressedSize is the length of the blob that was last passed to
// ApplyDiff() or Put(), as it was presented to us. If
// CompressedDigest is not set, this should be treated as if it were an
// uninitialized value.
CompressedSize int64 `json:"compressed-size,omitempty"`
// UncompressedDigest is the digest of the blob that was last passed to
// ApplyDiff() or Put(), after we decompressed it. Often referred to
// as a DiffID.
UncompressedDigest digest.Digest `json:"diff-digest,omitempty"`
// UncompressedSize is the length of the blob that was last passed to
// ApplyDiff() or Put(), after we decompressed it. If
// UncompressedDigest is not set, this should be treated as if it were
// an uninitialized value.
UncompressedSize int64 `json:"diff-size,omitempty"`
// CompressionType is the type of compression which we detected on the blob
// that was last passed to ApplyDiff() or Put().
CompressionType archive.Compression `json:"compression,omitempty"`
// Flags is arbitrary data about the layer.
Flags map[string]interface{} `json:"flags,omitempty"`
}
@ -75,12 +108,74 @@ type layerMountPoint struct {
MountCount int `json:"count"`
}
// DiffOptions override the default behavior of Diff() methods.
type DiffOptions struct {
// Compression, if set overrides the default compressor when generating a diff.
Compression *archive.Compression
}
// ROLayerStore wraps a graph driver, adding the ability to refer to layers by
// name, and keeping track of parent-child relationships, along with a list of
// all known layers.
type ROLayerStore interface {
ROFileBasedStore
ROMetadataStore
// Exists checks if a layer with the specified name or ID is known.
Exists(id string) bool
// Get retrieves information about a layer given an ID or name.
Get(id string) (*Layer, error)
// Status returns an slice of key-value pairs, suitable for human consumption,
// relaying whatever status information the underlying driver can share.
Status() ([][2]string, error)
// Changes returns a slice of Change structures, which contain a pathname
// (Path) and a description of what sort of change (Kind) was made by the
// layer (either ChangeModify, ChangeAdd, or ChangeDelete), relative to a
// specified layer. By default, the layer's parent is used as a reference.
Changes(from, to string) ([]archive.Change, error)
// Diff produces a tarstream which can be applied to a layer with the contents
// of the first layer to produce a layer with the contents of the second layer.
// By default, the parent of the second layer is used as the first
// layer, so it need not be specified. Options can be used to override
// default behavior, but are also not required.
Diff(from, to string, options *DiffOptions) (io.ReadCloser, error)
// DiffSize produces an estimate of the length of the tarstream which would be
// produced by Diff.
DiffSize(from, to string) (int64, error)
// Size produces a cached value for the uncompressed size of the layer,
// if one is known, or -1 if it is not known. If the layer can not be
// found, it returns an error.
Size(name string) (int64, error)
// Lookup attempts to translate a name to an ID. Most methods do this
// implicitly.
Lookup(name string) (string, error)
// LayersByCompressedDigest returns a slice of the layers with the
// specified compressed digest value recorded for them.
LayersByCompressedDigest(d digest.Digest) ([]Layer, error)
// LayersByUncompressedDigest returns a slice of the layers with the
// specified uncompressed digest value recorded for them.
LayersByUncompressedDigest(d digest.Digest) ([]Layer, error)
// Layers returns a slice of the known layers.
Layers() ([]Layer, error)
}
// LayerStore wraps a graph driver, adding the ability to refer to layers by
// name, and keeping track of parent-child relationships, along with a list of
// all known layers.
type LayerStore interface {
FileBasedStore
MetadataStore
ROLayerStore
RWFileBasedStore
RWMetadataStore
FlaggableStore
// Create creates a new layer, optionally giving it a specified ID rather than
@ -98,20 +193,10 @@ type LayerStore interface {
// Put combines the functions of CreateWithFlags and ApplyDiff.
Put(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}, diff archive.Reader) (*Layer, int64, error)
// Exists checks if a layer with the specified name or ID is known.
Exists(id string) bool
// Get retrieves information about a layer given an ID or name.
Get(id string) (*Layer, error)
// SetNames replaces the list of names associated with a layer with the
// supplied values.
SetNames(id string, names []string) error
// Status returns an slice of key-value pairs, suitable for human consumption,
// relaying whatever status information the underlying driver can share.
Status() ([][2]string, error)
// Delete deletes a layer with the specified name or ID.
Delete(id string) error
@ -126,48 +211,31 @@ type LayerStore interface {
// Unmount unmounts a layer when it is no longer in use.
Unmount(id string) error
// Changes returns a slice of Change structures, which contain a pathname
// (Path) and a description of what sort of change (Kind) was made by the
// layer (either ChangeModify, ChangeAdd, or ChangeDelete), relative to a
// specified layer. By default, the layer's parent is used as a reference.
Changes(from, to string) ([]archive.Change, error)
// Diff produces a tarstream which can be applied to a layer with the contents
// of the first layer to produce a layer with the contents of the second layer.
// By default, the parent of the second layer is used as the first
// layer, so it need not be specified.
Diff(from, to string) (io.ReadCloser, error)
// DiffSize produces an estimate of the length of the tarstream which would be
// produced by Diff.
DiffSize(from, to string) (int64, error)
// ApplyDiff reads a tarstream which was created by a previous call to Diff and
// applies its changes to a specified layer.
ApplyDiff(to string, diff archive.Reader) (int64, error)
// Lookup attempts to translate a name to an ID. Most methods do this
// implicitly.
Lookup(name string) (string, error)
// Layers returns a slice of the known layers.
Layers() ([]Layer, error)
}
type layerStore struct {
lockfile Locker
rundir string
driver drivers.Driver
layerdir string
layers []Layer
idindex *truncindex.TruncIndex
byid map[string]*Layer
byname map[string]*Layer
bymount map[string]*Layer
lockfile Locker
rundir string
driver drivers.Driver
layerdir string
layers []*Layer
idindex *truncindex.TruncIndex
byid map[string]*Layer
byname map[string]*Layer
bymount map[string]*Layer
bycompressedsum map[digest.Digest][]string
byuncompressedsum map[digest.Digest][]string
}
func (r *layerStore) Layers() ([]Layer, error) {
return r.layers, nil
layers := make([]Layer, len(r.layers))
for i := range r.layers {
layers[i] = *(r.layers[i])
}
return layers, nil
}
func (r *layerStore) mountspath() string {
@ -179,36 +247,41 @@ func (r *layerStore) layerspath() string {
}
func (r *layerStore) Load() error {
needSave := false
shouldSave := false
rpath := r.layerspath()
data, err := ioutil.ReadFile(rpath)
if err != nil && !os.IsNotExist(err) {
return err
}
layers := []Layer{}
layers := []*Layer{}
idlist := []string{}
ids := make(map[string]*Layer)
names := make(map[string]*Layer)
mounts := make(map[string]*Layer)
parents := make(map[string][]*Layer)
compressedsums := make(map[digest.Digest][]string)
uncompressedsums := make(map[digest.Digest][]string)
if err = json.Unmarshal(data, &layers); len(data) == 0 || err == nil {
for n, layer := range layers {
ids[layer.ID] = &layers[n]
ids[layer.ID] = layers[n]
idlist = append(idlist, layer.ID)
for _, name := range layer.Names {
if conflict, ok := names[name]; ok {
r.removeName(conflict, name)
needSave = true
shouldSave = true
}
names[name] = &layers[n]
names[name] = layers[n]
}
if pslice, ok := parents[layer.Parent]; ok {
parents[layer.Parent] = append(pslice, &layers[n])
} else {
parents[layer.Parent] = []*Layer{&layers[n]}
if layer.CompressedDigest != "" {
compressedsums[layer.CompressedDigest] = append(compressedsums[layer.CompressedDigest], layer.ID)
}
if layer.UncompressedDigest != "" {
uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID)
}
}
}
if shouldSave && !r.IsReadWrite() {
return errors.New("layer store assigns the same name to multiple layers")
}
mpath := r.mountspath()
data, err = ioutil.ReadFile(mpath)
if err != nil && !os.IsNotExist(err) {
@ -231,29 +304,35 @@ func (r *layerStore) Load() error {
r.byid = ids
r.byname = names
r.bymount = mounts
r.bycompressedsum = compressedsums
r.byuncompressedsum = uncompressedsums
err = nil
// Last step: try to remove anything that a previous user of this
// storage area marked for deletion but didn't manage to actually
// delete.
for _, layer := range r.layers {
if cleanup, ok := layer.Flags[incompleteFlag]; ok {
if b, ok := cleanup.(bool); ok && b {
err = r.Delete(layer.ID)
if err != nil {
break
// Last step: if we're writable, try to remove anything that a previous
// user of this storage area marked for deletion but didn't manage to
// actually delete.
if r.IsReadWrite() {
for _, layer := range r.layers {
if cleanup, ok := layer.Flags[incompleteFlag]; ok {
if b, ok := cleanup.(bool); ok && b {
err = r.Delete(layer.ID)
if err != nil {
break
}
shouldSave = true
}
needSave = true
}
}
}
if needSave {
r.Touch()
return r.Save()
if shouldSave {
return r.Save()
}
}
return err
}
func (r *layerStore) Save() error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify the layer store at %q", r.layerspath())
}
rpath := r.layerspath()
if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
return err
@ -283,6 +362,7 @@ func (r *layerStore) Save() error {
if err := ioutils.AtomicWriteFile(rpath, jldata, 0600); err != nil {
return err
}
defer r.Touch()
return ioutils.AtomicWriteFile(mpath, jmdata, 0600)
}
@ -314,6 +394,28 @@ func newLayerStore(rundir string, layerdir string, driver drivers.Driver) (Layer
return &rlstore, nil
}
func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (ROLayerStore, error) {
lockfile, err := GetROLockfile(filepath.Join(layerdir, "layers.lock"))
if err != nil {
return nil, err
}
lockfile.Lock()
defer lockfile.Unlock()
rlstore := layerStore{
lockfile: lockfile,
driver: driver,
rundir: rundir,
layerdir: layerdir,
byid: make(map[string]*Layer),
bymount: make(map[string]*Layer),
byname: make(map[string]*Layer),
}
if err := rlstore.Load(); err != nil {
return nil, err
}
return &rlstore, nil
}
func (r *layerStore) lookup(id string) (*Layer, bool) {
if layer, ok := r.byid[id]; ok {
return layer, ok
@ -326,7 +428,24 @@ func (r *layerStore) lookup(id string) (*Layer, bool) {
return nil, false
}
func (r *layerStore) Size(name string) (int64, error) {
layer, ok := r.lookup(name)
if !ok {
return -1, ErrLayerUnknown
}
// We use the presence of a non-empty digest as an indicator that the size value was intentionally set, and that
// a zero value is not just present because it was never set to anything else (which can happen if the layer was
// created by a version of this library that didn't keep track of digest and size information).
if layer.UncompressedDigest != "" {
return layer.UncompressedSize, nil
}
return -1, nil
}
func (r *layerStore) ClearFlag(id string, flag string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to clear flags on layers at %q", r.layerspath())
}
layer, ok := r.lookup(id)
if !ok {
return ErrLayerUnknown
@ -336,6 +455,9 @@ func (r *layerStore) ClearFlag(id string, flag string) error {
}
func (r *layerStore) SetFlag(id string, flag string, value interface{}) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to set flags on layers at %q", r.layerspath())
}
layer, ok := r.lookup(id)
if !ok {
return ErrLayerUnknown
@ -349,6 +471,9 @@ func (r *layerStore) Status() ([][2]string, error) {
}
func (r *layerStore) Put(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}, diff archive.Reader) (layer *Layer, size int64, err error) {
if !r.IsReadWrite() {
return nil, -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new layers at %q", r.layerspath())
}
size = -1
if err := os.MkdirAll(r.rundir, 0700); err != nil {
return nil, -1, err
@ -383,15 +508,15 @@ func (r *layerStore) Put(id, parent string, names []string, mountLabel string, o
err = r.driver.Create(id, parent, mountLabel, options)
}
if err == nil {
newLayer := Layer{
layer = &Layer{
ID: id,
Parent: parent,
Names: names,
MountLabel: mountLabel,
Created: time.Now().UTC(),
Flags: make(map[string]interface{}),
}
r.layers = append(r.layers, newLayer)
layer = &r.layers[len(r.layers)-1]
r.layers = append(r.layers, layer)
r.idindex.Add(id)
r.byid[id] = layer
for _, name := range names {
@ -441,6 +566,9 @@ func (r *layerStore) Create(id, parent string, names []string, mountLabel string
}
func (r *layerStore) Mount(id, mountLabel string) (string, error) {
if !r.IsReadWrite() {
return "", errors.Wrapf(ErrStoreIsReadOnly, "not allowed to update mount locations for layers at %q", r.mountspath())
}
layer, ok := r.lookup(id)
if !ok {
return "", ErrLayerUnknown
@ -466,6 +594,9 @@ func (r *layerStore) Mount(id, mountLabel string) (string, error) {
}
func (r *layerStore) Unmount(id string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to update mount locations for layers at %q", r.mountspath())
}
layer, ok := r.lookup(id)
if !ok {
layerByMount, ok := r.bymount[filepath.Clean(id)]
@ -495,6 +626,9 @@ func (r *layerStore) removeName(layer *Layer, name string) {
}
func (r *layerStore) SetNames(id string, names []string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to change layer name assignments at %q", r.layerspath())
}
if layer, ok := r.lookup(id); ok {
for _, name := range layer.Names {
delete(r.byname, name)
@ -519,6 +653,9 @@ func (r *layerStore) Metadata(id string) (string, error) {
}
func (r *layerStore) SetMetadata(id, metadata string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify layer metadata at %q", r.layerspath())
}
if layer, ok := r.lookup(id); ok {
layer.Metadata = metadata
return r.Save()
@ -531,6 +668,9 @@ func (r *layerStore) tspath(id string) string {
}
func (r *layerStore) Delete(id string) error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to delete layers at %q", r.layerspath())
}
layer, ok := r.lookup(id)
if !ok {
return ErrLayerUnknown
@ -549,13 +689,21 @@ func (r *layerStore) Delete(id string) error {
if layer.MountPoint != "" {
delete(r.bymount, layer.MountPoint)
}
newLayers := []Layer{}
for _, candidate := range r.layers {
if candidate.ID != id {
newLayers = append(newLayers, candidate)
toDeleteIndex := -1
for i, candidate := range r.layers {
if candidate.ID == id {
toDeleteIndex = i
break
}
}
if toDeleteIndex != -1 {
// delete the layer at toDeleteIndex
if toDeleteIndex == len(r.layers)-1 {
r.layers = r.layers[:len(r.layers)-1]
} else {
r.layers = append(r.layers[:toDeleteIndex], r.layers[toDeleteIndex+1:]...)
}
}
r.layers = newLayers
if err = r.Save(); err != nil {
return err
}
@ -583,6 +731,9 @@ func (r *layerStore) Get(id string) (*Layer, error) {
}
func (r *layerStore) Wipe() error {
if !r.IsReadWrite() {
return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to delete layers at %q", r.layerspath())
}
ids := []string{}
for id := range r.byid {
ids = append(ids, id)
@ -657,48 +808,64 @@ func (r *layerStore) newFileGetter(id string) (drivers.FileGetCloser, error) {
}, nil
}
func (r *layerStore) Diff(from, to string) (io.ReadCloser, error) {
func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) {
var metadata storage.Unpacker
from, to, toLayer, err := r.findParentAndLayer(from, to)
if err != nil {
return nil, ErrLayerUnknown
}
compression := archive.Uncompressed
if cflag, ok := toLayer.Flags[compressionFlag]; ok {
if ctype, ok := cflag.(float64); ok {
compression = archive.Compression(ctype)
} else if ctype, ok := cflag.(archive.Compression); ok {
compression = archive.Compression(ctype)
}
// Default to applying the type of compression that we noted was used
// for the layerdiff when it was applied.
compression := toLayer.CompressionType
// If a particular compression type (or no compression) was selected,
// use that instead.
if options != nil && options.Compression != nil {
compression = *options.Compression
}
maybeCompressReadCloser := func(rc io.ReadCloser) (io.ReadCloser, error) {
// Depending on whether or not compression is desired, return either the
// passed-in ReadCloser, or a new one that provides its readers with a
// compressed version of the data that the original would have provided
// to its readers.
if compression == archive.Uncompressed {
return rc, nil
}
preader, pwriter := io.Pipe()
compressor, err := archive.CompressStream(pwriter, compression)
if err != nil {
rc.Close()
pwriter.Close()
preader.Close()
return nil, err
}
go func() {
defer pwriter.Close()
defer compressor.Close()
defer rc.Close()
io.Copy(compressor, rc)
}()
return preader, nil
}
if from != toLayer.Parent {
diff, err := r.driver.Diff(to, from)
if err == nil && (compression != archive.Uncompressed) {
preader, pwriter := io.Pipe()
compressor, err := archive.CompressStream(pwriter, compression)
if err != nil {
diff.Close()
pwriter.Close()
return nil, err
}
go func() {
io.Copy(compressor, diff)
diff.Close()
compressor.Close()
pwriter.Close()
}()
diff = preader
if err != nil {
return nil, err
}
return diff, err
return maybeCompressReadCloser(diff)
}
tsfile, err := os.Open(r.tspath(to))
if err != nil {
if os.IsNotExist(err) {
return r.driver.Diff(to, from)
if !os.IsNotExist(err) {
return nil, err
}
return nil, err
diff, err := r.driver.Diff(to, from)
if err != nil {
return nil, err
}
return maybeCompressReadCloser(diff)
}
defer tsfile.Close()
@ -720,33 +887,16 @@ func (r *layerStore) Diff(from, to string) (io.ReadCloser, error) {
return nil, err
}
var stream io.ReadCloser
if compression != archive.Uncompressed {
preader, pwriter := io.Pipe()
compressor, err := archive.CompressStream(pwriter, compression)
if err != nil {
fgetter.Close()
pwriter.Close()
preader.Close()
return nil, err
}
go func() {
asm.WriteOutputTarStream(fgetter, metadata, compressor)
compressor.Close()
pwriter.Close()
}()
stream = preader
} else {
stream = asm.NewOutputTarStream(fgetter, metadata)
}
return ioutils.NewReadCloserWrapper(stream, func() error {
err1 := stream.Close()
tarstream := asm.NewOutputTarStream(fgetter, metadata)
rc := ioutils.NewReadCloserWrapper(tarstream, func() error {
err1 := tarstream.Close()
err2 := fgetter.Close()
if err2 == nil {
return err1
}
return err2
}), nil
})
return maybeCompressReadCloser(rc)
}
func (r *layerStore) DiffSize(from, to string) (size int64, err error) {
@ -758,6 +908,10 @@ func (r *layerStore) DiffSize(from, to string) (size int64, err error) {
}
func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err error) {
if !r.IsReadWrite() {
return -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify layer contents at %q", r.layerspath())
}
layer, ok := r.lookup(to)
if !ok {
return -1, ErrLayerUnknown
@ -770,7 +924,9 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err
}
compression := archive.DetectCompression(header[:n])
defragmented := io.MultiReader(bytes.NewBuffer(header[:n]), diff)
compressedDigest := digest.Canonical.Digester()
compressedCounter := ioutils.NewWriteCounter(compressedDigest.Hash())
defragmented := io.TeeReader(io.MultiReader(bytes.NewBuffer(header[:n]), diff), compressedCounter)
tsdata := bytes.Buffer{}
compressor, err := gzip.NewWriterLevel(&tsdata, gzip.BestSpeed)
@ -778,15 +934,20 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err
compressor = gzip.NewWriter(&tsdata)
}
metadata := storage.NewJSONPacker(compressor)
decompressed, err := archive.DecompressStream(defragmented)
uncompressed, err := archive.DecompressStream(defragmented)
if err != nil {
return -1, err
}
payload, err := asm.NewInputTarStream(decompressed, metadata, storage.NewDiscardFilePutter())
uncompressedDigest := digest.Canonical.Digester()
uncompressedCounter := ioutils.NewWriteCounter(uncompressedDigest.Hash())
payload, err := asm.NewInputTarStream(io.TeeReader(uncompressed, uncompressedCounter), metadata, storage.NewDiscardFilePutter())
if err != nil {
return -1, err
}
size, err = r.driver.ApplyDiff(layer.ID, layer.Parent, payload)
if err != nil {
return -1, err
}
compressor.Close()
if err == nil {
if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0700); err != nil {
@ -797,15 +958,57 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err
}
}
if compression != archive.Uncompressed {
layer.Flags[compressionFlag] = compression
} else {
delete(layer.Flags, compressionFlag)
updateDigestMap := func(m *map[digest.Digest][]string, oldvalue, newvalue digest.Digest, id string) {
var newList []string
if oldvalue != "" {
for _, value := range (*m)[oldvalue] {
if value != id {
newList = append(newList, value)
}
}
if len(newList) > 0 {
(*m)[oldvalue] = newList
} else {
delete(*m, oldvalue)
}
}
if newvalue != "" {
(*m)[newvalue] = append((*m)[newvalue], id)
}
}
updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, compressedDigest.Digest(), layer.ID)
layer.CompressedDigest = compressedDigest.Digest()
layer.CompressedSize = compressedCounter.Count
updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, uncompressedDigest.Digest(), layer.ID)
layer.UncompressedDigest = uncompressedDigest.Digest()
layer.UncompressedSize = uncompressedCounter.Count
layer.CompressionType = compression
err = r.Save()
return size, err
}
func (r *layerStore) layersByDigestMap(m map[digest.Digest][]string, d digest.Digest) ([]Layer, error) {
var layers []Layer
for _, layerID := range m[d] {
layer, ok := r.lookup(layerID)
if !ok {
return nil, ErrLayerUnknown
}
layers = append(layers, *layer)
}
return layers, nil
}
func (r *layerStore) LayersByCompressedDigest(d digest.Digest) ([]Layer, error) {
return r.layersByDigestMap(r.bycompressedsum, d)
}
func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) {
return r.layersByDigestMap(r.byuncompressedsum, d)
}
func (r *layerStore) Lock() {
r.lockfile.Lock()
}
@ -822,6 +1025,10 @@ func (r *layerStore) Modified() (bool, error) {
return r.lockfile.Modified()
}
func (r *layerStore) IsReadWrite() bool {
return r.lockfile.IsReadWrite()
}
func (r *layerStore) TouchedSince(when time.Time) bool {
return r.lockfile.TouchedSince(when)
}

View file

@ -1,14 +1,15 @@
package storage
import (
"fmt"
"os"
"path/filepath"
"sync"
"time"
"golang.org/x/sys/unix"
"github.com/containers/storage/pkg/stringid"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
// A Locker represents a file lock where the file is used to cache an
@ -27,43 +28,80 @@ type Locker interface {
// TouchedSince() checks if the most recent writer modified the file (likely using Touch()) after the specified time.
TouchedSince(when time.Time) bool
// IsReadWrite() checks if the lock file is read-write
IsReadWrite() bool
}
type lockfile struct {
mu sync.Mutex
file string
fd uintptr
lw string
mu sync.Mutex
file string
fd uintptr
lw string
locktype int16
}
var (
lockfiles map[string]*lockfile
lockfilesLock sync.Mutex
// ErrLockReadOnly indicates that the caller only took a read-only lock, and is not allowed to write
ErrLockReadOnly = errors.New("lock is not a read-write lock")
)
// GetLockfile opens a lock file, creating it if necessary. The Locker object
// return will be returned unlocked.
// GetLockfile opens a read-write lock file, creating it if necessary. The
// Locker object it returns will be returned unlocked.
func GetLockfile(path string) (Locker, error) {
lockfilesLock.Lock()
defer lockfilesLock.Unlock()
if lockfiles == nil {
lockfiles = make(map[string]*lockfile)
}
if locker, ok := lockfiles[filepath.Clean(path)]; ok {
cleanPath := filepath.Clean(path)
if locker, ok := lockfiles[cleanPath]; ok {
if !locker.IsReadWrite() {
return nil, errors.Wrapf(ErrLockReadOnly, "lock %q is a read-only lock", cleanPath)
}
return locker, nil
}
fd, err := unix.Open(filepath.Clean(path), os.O_RDWR|os.O_CREATE, unix.S_IRUSR|unix.S_IWUSR)
fd, err := unix.Open(cleanPath, os.O_RDWR|os.O_CREATE, unix.S_IRUSR|unix.S_IWUSR)
if err != nil {
return nil, err
return nil, errors.Wrapf(err, "error opening %q", cleanPath)
}
locker := &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID()}
unix.CloseOnExec(fd)
locker := &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_WRLCK}
lockfiles[filepath.Clean(path)] = locker
return locker, nil
}
// GetROLockfile opens a read-only lock file. The Locker object it returns
// will be returned unlocked.
func GetROLockfile(path string) (Locker, error) {
lockfilesLock.Lock()
defer lockfilesLock.Unlock()
if lockfiles == nil {
lockfiles = make(map[string]*lockfile)
}
cleanPath := filepath.Clean(path)
if locker, ok := lockfiles[cleanPath]; ok {
if locker.IsReadWrite() {
return nil, fmt.Errorf("lock %q is a read-write lock", cleanPath)
}
return locker, nil
}
fd, err := unix.Open(cleanPath, os.O_RDONLY, 0)
if err != nil {
return nil, errors.Wrapf(err, "error opening %q", cleanPath)
}
unix.CloseOnExec(fd)
locker := &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_RDLCK}
lockfiles[filepath.Clean(path)] = locker
return locker, nil
}
// Lock locks the lock file
func (l *lockfile) Lock() {
lk := unix.Flock_t{
Type: unix.F_WRLCK,
Type: l.locktype,
Whence: int16(os.SEEK_SET),
Start: 0,
Len: 0,
@ -75,6 +113,7 @@ func (l *lockfile) Lock() {
}
}
// Unlock unlocks the lock file
func (l *lockfile) Unlock() {
lk := unix.Flock_t{
Type: unix.F_UNLCK,
@ -89,6 +128,7 @@ func (l *lockfile) Unlock() {
l.mu.Unlock()
}
// Touch updates the lock file with the UID of the user
func (l *lockfile) Touch() error {
l.lw = stringid.GenerateRandomID()
id := []byte(l.lw)
@ -110,6 +150,7 @@ func (l *lockfile) Touch() error {
return nil
}
// Modified indicates if the lock file has been updated since the last time it was loaded
func (l *lockfile) Modified() (bool, error) {
id := []byte(l.lw)
_, err := unix.Seek(int(l.fd), 0, os.SEEK_SET)
@ -128,6 +169,7 @@ func (l *lockfile) Modified() (bool, error) {
return l.lw != lw, nil
}
// TouchedSince indicates if the lock file has been touched since the specified time
func (l *lockfile) TouchedSince(when time.Time) bool {
st := unix.Stat_t{}
err := unix.Fstat(int(l.fd), &st)
@ -137,3 +179,8 @@ func (l *lockfile) TouchedSince(when time.Time) bool {
touched := time.Unix(statTMtimeUnix(st))
return when.Before(touched)
}
// IsRWLock indicates if the lock file is a read-write lock
func (l *lockfile) IsReadWrite() bool {
return (l.locktype == unix.F_WRLCK)
}

View file

@ -38,7 +38,15 @@ func getNextFreeLoopbackIndex() (int, error) {
return index, err
}
func openNextAvailableLoopback(index int, sparseFile *os.File) (loopFile *os.File, err error) {
func openNextAvailableLoopback(index int, sparseName string, sparseFile *os.File) (loopFile *os.File, err error) {
// Read information about the loopback file.
var st syscall.Stat_t
err = syscall.Fstat(int(sparseFile.Fd()), &st)
if err != nil {
logrus.Errorf("Error reading information about loopback file %s: %v", sparseName, err)
return nil, ErrAttachLoopbackDevice
}
// Start looking for a free /dev/loop
for {
target := fmt.Sprintf("/dev/loop%d", index)
@ -77,6 +85,18 @@ func openNextAvailableLoopback(index int, sparseFile *os.File) (loopFile *os.Fil
// Otherwise, we keep going with the loop
continue
}
// Check if the loopback driver and underlying filesystem agree on the loopback file's
// device and inode numbers.
dev, ino, err := getLoopbackBackingFile(loopFile)
if err != nil {
logrus.Errorf("Error getting loopback backing file: %s", err)
return nil, ErrGetLoopbackBackingFile
}
if dev != st.Dev || ino != st.Ino {
logrus.Errorf("Loopback device and filesystem disagree on device/inode for %q: %#x(%d):%#x(%d) vs %#x(%d):%#x(%d)", sparseName, dev, dev, ino, ino, st.Dev, st.Dev, st.Ino, st.Ino)
}
// In case of success, we finished. Break the loop.
break
}
@ -110,7 +130,7 @@ func AttachLoopDevice(sparseName string) (loop *os.File, err error) {
}
defer sparseFile.Close()
loopFile, err := openNextAvailableLoopback(startIndex, sparseFile)
loopFile, err := openNextAvailableLoopback(startIndex, sparseName, sparseFile)
if err != nil {
return nil, err
}

View file

@ -15,8 +15,8 @@ import (
var (
// ErrNotFound plugin not found
ErrNotFound = errors.New("plugin not found")
socketsPath = "/run/oci-storage/plugins"
specsPaths = []string{"/etc/oci-storage/plugins", "/usr/lib/oci-storage/plugins"}
socketsPath = "/run/containers/storage/plugins"
specsPaths = []string{"/etc/containers/storage/plugins", "/usr/lib/containers/storage/plugins"}
)
// localRegistry defines a registry that is local (using unix socket).

View file

@ -3,10 +3,11 @@
//
// Storage discovers plugins by looking for them in the plugin directory whenever
// a user or container tries to use one by name. UNIX domain socket files must
// be located under /run/oci-storage/plugins, whereas spec files can be located
// either under /etc/oci-storage/plugins or /usr/lib/oci-storage/plugins. This
// is handled by the Registry interface, which lets you list all plugins or get
// a plugin by its name if it exists.
// be located under /run/containers/storage/plugins, whereas spec files can be
// located either under /etc/containers/storage/plugins or
// /usr/lib/containers/storage/plugins. This is handled by the Registry
// interface, which lets you list all plugins or get a plugin by its name if it
// exists.
//
// The plugins need to implement an HTTP server and bind this to the UNIX socket
// or the address specified in the spec files.

View file

@ -1,13 +0,0 @@
// +build !containersstorageautogen
// Package storageversion is auto-generated at build-time
package storageversion
// Default build-time variable for library-import.
// This file is overridden on build with build-time informations.
const (
GitCommit string = "library-import"
Version string = "library-import"
BuildTime string = "library-import"
IAmStatic string = "library-import"
)

File diff suppressed because it is too large Load diff

View file

@ -1,3 +1,4 @@
github.com/BurntSushi/toml master
github.com/Microsoft/go-winio 307e919c663683a9000576fdc855acaf9534c165
github.com/Microsoft/hcsshim 0f615c198a84e0344b4ed49c464d8833d4648dfc
github.com/Sirupsen/logrus 61e43dc76f7ee59a82bdf3d71033dc12bea4c77d
@ -7,9 +8,11 @@ github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52
github.com/go-check/check 20d25e2804050c1cd24a7eea1e7a6447dd0e74ec
github.com/mattn/go-shellwords 753a2322a99f87c0eff284980e77f53041555bc6
github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062
github.com/opencontainers/go-digest master
github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07
github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d
github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9
github.com/pkg/errors master
github.com/tchap/go-patricia v2.2.6
github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721
github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3