diff --git a/vendor.conf b/vendor.conf index f37a2ee6..b61ce8e2 100644 --- a/vendor.conf +++ b/vendor.conf @@ -6,7 +6,7 @@ k8s.io/apiserver release-1.6 https://github.com/kubernetes/apiserver # github.com/Sirupsen/logrus v0.11.5 github.com/containers/image b36d6535410088370aaaee7ec8522863b5e43489 -github.com/containers/storage 2c75d14b978bff468e7d5ec3ff8a003eca443209 +github.com/containers/storage 74bc9c18a31d0e6fb1a11ce4563159b944a73d2e github.com/containernetworking/cni v0.4.0 google.golang.org/grpc v1.0.1-GA https://github.com/grpc/grpc-go github.com/opencontainers/selinux v1.0.0-rc1 diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index 90a0bc0b..0908bdd1 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -50,6 +50,12 @@ type Container struct { // that has been stored, if they're known. BigDataSizes map[string]int64 `json:"big-data-sizes,omitempty"` + // Created is the datestamp for when this container was created. Older + // versions of the library did not track this information, so callers + // will likely want to use the IsZero() method to verify that a value + // is set before using it. + Created time.Time `json:"created,omitempty"` + Flags map[string]interface{} `json:"flags,omitempty"` } @@ -93,7 +99,7 @@ type ContainerStore interface { type containerStore struct { lockfile Locker dir string - containers []Container + containers []*Container idindex *truncindex.TruncIndex byid map[string]*Container bylayer map[string]*Container @@ -101,7 +107,11 @@ type containerStore struct { } func (r *containerStore) Containers() ([]Container, error) { - return r.containers, nil + containers := make([]Container, len(r.containers)) + for i := range r.containers { + containers[i] = *(r.containers[i]) + } + return containers, nil } func (r *containerStore) containerspath() string { @@ -123,7 +133,7 @@ func (r *containerStore) Load() error { if err != nil && !os.IsNotExist(err) { return err } - containers := []Container{} + containers := []*Container{} layers := make(map[string]*Container) idlist := []string{} ids := make(map[string]*Container) @@ -131,14 +141,14 @@ func (r *containerStore) Load() error { if err = json.Unmarshal(data, &containers); len(data) == 0 || err == nil { for n, container := range containers { idlist = append(idlist, container.ID) - ids[container.ID] = &containers[n] - layers[container.LayerID] = &containers[n] + ids[container.ID] = containers[n] + layers[container.LayerID] = containers[n] for _, name := range container.Names { if conflict, ok := names[name]; ok { r.removeName(conflict, name) needSave = true } - names[name] = &containers[n] + names[name] = containers[n] } } } @@ -148,7 +158,6 @@ func (r *containerStore) Load() error { r.bylayer = layers r.byname = names if needSave { - r.Touch() return r.Save() } return nil @@ -163,6 +172,7 @@ func (r *containerStore) Save() error { if err != nil { return err } + defer r.Touch() return ioutils.AtomicWriteFile(rpath, jdata, 0600) } @@ -179,7 +189,7 @@ func newContainerStore(dir string) (ContainerStore, error) { cstore := containerStore{ lockfile: lockfile, dir: dir, - containers: []Container{}, + containers: []*Container{}, byid: make(map[string]*Container), bylayer: make(map[string]*Container), byname: make(map[string]*Container), @@ -241,7 +251,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat } } if err == nil { - newContainer := Container{ + container = &Container{ ID: id, Names: names, ImageID: image, @@ -249,10 +259,10 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat Metadata: metadata, BigDataNames: []string{}, BigDataSizes: make(map[string]int64), + Created: time.Now().UTC(), Flags: make(map[string]interface{}), } - r.containers = append(r.containers, newContainer) - container = &r.containers[len(r.containers)-1] + r.containers = append(r.containers, container) r.byid[id] = container r.idindex.Add(id) r.bylayer[layer] = container @@ -306,10 +316,11 @@ func (r *containerStore) Delete(id string) error { return ErrContainerUnknown } id = container.ID - newContainers := []Container{} - for _, candidate := range r.containers { - if candidate.ID != id { - newContainers = append(newContainers, candidate) + toDeleteIndex := -1 + for i, candidate := range r.containers { + if candidate.ID == id { + toDeleteIndex = i + break } } delete(r.byid, id) @@ -318,7 +329,14 @@ func (r *containerStore) Delete(id string) error { for _, name := range container.Names { delete(r.byname, name) } - r.containers = newContainers + if toDeleteIndex != -1 { + // delete the container at toDeleteIndex + if toDeleteIndex == len(r.containers)-1 { + r.containers = r.containers[:len(r.containers)-1] + } else { + r.containers = append(r.containers[:toDeleteIndex], r.containers[toDeleteIndex+1:]...) + } + } if err := r.Save(); err != nil { return err } @@ -437,6 +455,10 @@ func (r *containerStore) Modified() (bool, error) { return r.lockfile.Modified() } +func (r *containerStore) IsReadWrite() bool { + return r.lockfile.IsReadWrite() +} + func (r *containerStore) TouchedSince(when time.Time) bool { return r.lockfile.TouchedSince(when) } diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index 8caa91fe..d610333d 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -372,6 +372,12 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { }) } +// AdditionalImageStores returns additional image stores supported by the driver +func (a *Driver) AdditionalImageStores() []string { + var imageStores []string + return imageStores +} + type fileGetNilCloser struct { storage.FileGetter } diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index 5bcee11f..6541eda3 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -518,3 +518,9 @@ func (d *Driver) Exists(id string) bool { _, err := os.Stat(dir) return err == nil } + +// AdditionalImageStores returns additional image stores supported by the driver +func (d *Driver) AdditionalImageStores() []string { + var imageStores []string + return imageStores +} diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index caa7474c..b2332b7f 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -28,7 +28,6 @@ import ( "github.com/containers/storage/pkg/loopback" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" - "github.com/containers/storage/storageversion" "github.com/docker/go-units" "github.com/opencontainers/selinux/go-selinux/label" @@ -1668,17 +1667,17 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { } // https://github.com/docker/docker/issues/4036 - if supported := devicemapper.UdevSetSyncSupport(true); !supported { - if storageversion.IAmStatic == "true" { - logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option") - } else { - logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option") - } - - if !devices.overrideUdevSyncCheck { - return graphdriver.ErrNotSupported - } - } + // if supported := devicemapper.UdevSetSyncSupport(true); !supported { + // if storageversion.IAmStatic == "true" { + // logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a dynamic binary to use devicemapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option") + // } else { + // logrus.Errorf("devmapper: Udev sync is not supported. This will lead to data loss and unexpected behavior. Install a more recent version of libdevmapper or select a different storage driver. For more information, see https://docs.docker.com/engine/reference/commandline/daemon/#daemon-storage-driver-option") + // } + // + // if !devices.overrideUdevSyncCheck { + // return graphdriver.ErrNotSupported + // } + // } //create the root dir of the devmapper driver ownership to match this //daemon's remapped root uid/gid so containers can start properly diff --git a/vendor/github.com/containers/storage/drivers/devmapper/driver.go b/vendor/github.com/containers/storage/drivers/devmapper/driver.go index a1174240..3b584c06 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/driver.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/driver.go @@ -224,3 +224,9 @@ func (d *Driver) Put(id string) error { func (d *Driver) Exists(id string) bool { return d.DeviceSet.HasDevice(id) } + +// AdditionalImageStores returns additional image stores supported by the driver +func (d *Driver) AdditionalImageStores() []string { + var imageStores []string + return imageStores +} diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index cdf91d02..983327fd 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -74,6 +74,8 @@ type ProtoDriver interface { // held by the driver, e.g., unmounting all layered filesystems // known to this driver. Cleanup() error + // AdditionalImageStores returns additional image stores supported by the driver + AdditionalImageStores() []string } // Driver is the interface for layered/snapshot file system drivers. diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 6c1642cb..d0f73e3e 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strconv" "strings" "syscall" @@ -82,6 +83,7 @@ type Driver struct { uidMaps []idtools.IDMap gidMaps []idtools.IDMap ctr *graphdriver.RefCounter + opts *overlayOptions } var backingFs = "" @@ -149,6 +151,7 @@ func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtool uidMaps: uidMaps, gidMaps: gidMaps, ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)), + opts: opts, } return d, nil @@ -170,6 +173,7 @@ func InitAsOverlay2(home string, options []string, uidMaps, gidMaps []idtools.ID type overlayOptions struct { overrideKernelCheck bool + imageStores []string } func parseOptions(options []string) (*overlayOptions, error) { @@ -186,6 +190,22 @@ func parseOptions(options []string) (*overlayOptions, error) { if err != nil { return nil, err } + case "overlay.imagestore": + // Additional read only image stores to use for lower paths + for _, store := range strings.Split(val, ",") { + store = filepath.Clean(store) + if !filepath.IsAbs(store) { + return nil, fmt.Errorf("overlay: image path %q is not absolute. Can not be relative", store) + } + st, err := os.Stat(store) + if err != nil { + return nil, fmt.Errorf("overlay: Can't stat imageStore dir %s: %v", store, err) + } + if !st.IsDir() { + return nil, fmt.Errorf("overlay: image path %q must be a directory", store) + } + o.imageStores = append(o.imageStores, store) + } default: return nil, fmt.Errorf("overlay: Unknown option %s", key) } @@ -357,8 +377,18 @@ func (d *Driver) getLower(parent string) (string, error) { return strings.Join(lowers, ":"), nil } -func (d *Driver) dir(id string) string { - return path.Join(d.home, id) +func (d *Driver) dir(val string) string { + newpath := path.Join(d.home, val) + if _, err := os.Stat(newpath); err != nil { + for _, p := range d.AdditionalImageStores() { + l := path.Join(p, d.name, val) + _, err = os.Stat(l) + if err == nil { + return l + } + } + } + return newpath } func (d *Driver) getLowerDirs(id string) ([]string, error) { @@ -366,11 +396,12 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) { lowers, err := ioutil.ReadFile(path.Join(d.dir(id), lowerFile)) if err == nil { for _, s := range strings.Split(string(lowers), ":") { - lp, err := os.Readlink(path.Join(d.home, s)) + lower := d.dir(s) + lp, err := os.Readlink(lower) if err != nil { return nil, err } - lowersArray = append(lowersArray, path.Clean(path.Join(d.home, "link", lp))) + lowersArray = append(lowersArray, path.Clean(d.dir(path.Join("link", lp)))) } } else if !os.IsNotExist(err) { return nil, err @@ -411,6 +442,31 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) { return "", err } + newlowers := "" + for _, l := range strings.Split(string(lowers), ":") { + lower := "" + newpath := path.Join(d.home, l) + if _, err := os.Stat(newpath); err != nil { + for _, p := range d.AdditionalImageStores() { + lower = path.Join(p, d.name, l) + if _, err2 := os.Stat(lower); err2 == nil { + break + } + lower = "" + } + if lower == "" { + return "", fmt.Errorf("Can't stat lower layer %q: %v", newpath, err) + } + } else { + lower = l + } + if newlowers == "" { + newlowers = lower + } else { + newlowers = newlowers + ":" + lower + } + } + mergedDir := path.Join(dir, "merged") if count := d.ctr.Increment(mergedDir); count > 1 { return mergedDir, nil @@ -424,7 +480,7 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) { }() workDir := path.Join(dir, "work") - opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", string(lowers), path.Join(id, "diff"), path.Join(id, "work")) + opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", newlowers, path.Join(id, "diff"), path.Join(id, "work")) mountLabel = label.FormatMountLabel(opts, mountLabel) if len(mountLabel) > syscall.Getpagesize() { return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountLabel)) @@ -527,3 +583,8 @@ func (d *Driver) Changes(id, parent string) ([]archive.Change, error) { return archive.OverlayChanges(layers, diffPath) } + +// AdditionalImageStores returns additional image stores supported by the driver +func (d *Driver) AdditionalImageStores() []string { + return d.opts.imageStores +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index 5dd934fd..ff7a88f1 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -143,3 +143,9 @@ func (d *Driver) Exists(id string) bool { _, err := os.Stat(d.dir(id)) return err == nil } + +// AdditionalImageStores returns additional image stores supported by the driver +func (d *Driver) AdditionalImageStores() []string { + var imageStores []string + return imageStores +} diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs.go b/vendor/github.com/containers/storage/drivers/zfs/zfs.go index 8fd17e6c..ec708a74 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs.go @@ -403,3 +403,9 @@ func (d *Driver) Exists(id string) bool { defer d.Unlock() return d.filesystemsCache[d.zfsPath(id)] == true } + +// AdditionalImageStores returns additional image stores supported by the driver +func (d *Driver) AdditionalImageStores() []string { + var imageStores []string + return imageStores +} diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index 6d9a7b58..fe17f631 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -2,7 +2,6 @@ package storage import ( "encoding/json" - "errors" "io/ioutil" "os" "path/filepath" @@ -11,6 +10,7 @@ import ( "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/stringid" "github.com/containers/storage/pkg/truncindex" + "github.com/pkg/errors" ) var ( @@ -46,24 +46,20 @@ type Image struct { // that has been stored, if they're known. BigDataSizes map[string]int64 `json:"big-data-sizes,omitempty"` + // Created is the datestamp for when this image was created. Older + // versions of the library did not track this information, so callers + // will likely want to use the IsZero() method to verify that a value + // is set before using it. + Created time.Time `json:"created,omitempty"` + Flags map[string]interface{} `json:"flags,omitempty"` } -// ImageStore provides bookkeeping for information about Images. -type ImageStore interface { - FileBasedStore - MetadataStore - BigDataStore - FlaggableStore - - // Create creates an image that has a specified ID (or a random one) and - // optional names, using the specified layer as its topmost (hopefully - // read-only) layer. That layer can be referenced by multiple images. - Create(id string, names []string, layer, metadata string) (*Image, error) - - // SetNames replaces the list of names associated with an image with the - // supplied values. - SetNames(id string, names []string) error +// ROImageStore provides bookkeeping for information about Images. +type ROImageStore interface { + ROFileBasedStore + ROMetadataStore + ROBigDataStore // Exists checks if there is an image with the given ID or name. Exists(id string) bool @@ -71,12 +67,6 @@ type ImageStore interface { // Get retrieves information about an image given an ID or name. Get(id string) (*Image, error) - // Delete removes the record of the image. - Delete(id string) error - - // Wipe removes records of all images. - Wipe() error - // Lookup attempts to translate a name to an ID. Most methods do this // implicitly. Lookup(name string) (string, error) @@ -85,17 +75,45 @@ type ImageStore interface { Images() ([]Image, error) } +// ImageStore provides bookkeeping for information about Images. +type ImageStore interface { + ROImageStore + RWFileBasedStore + RWMetadataStore + RWBigDataStore + FlaggableStore + + // Create creates an image that has a specified ID (or a random one) and + // optional names, using the specified layer as its topmost (hopefully + // read-only) layer. That layer can be referenced by multiple images. + Create(id string, names []string, layer, metadata string, created time.Time) (*Image, error) + + // SetNames replaces the list of names associated with an image with the + // supplied values. + SetNames(id string, names []string) error + + // Delete removes the record of the image. + Delete(id string) error + + // Wipe removes records of all images. + Wipe() error +} + type imageStore struct { lockfile Locker dir string - images []Image + images []*Image idindex *truncindex.TruncIndex byid map[string]*Image byname map[string]*Image } func (r *imageStore) Images() ([]Image, error) { - return r.images, nil + images := make([]Image, len(r.images)) + for i := range r.images { + images[i] = *(r.images[i]) + } + return images, nil } func (r *imageStore) imagespath() string { @@ -111,41 +129,46 @@ func (r *imageStore) datapath(id, key string) string { } func (r *imageStore) Load() error { - needSave := false + shouldSave := false rpath := r.imagespath() data, err := ioutil.ReadFile(rpath) if err != nil && !os.IsNotExist(err) { return err } - images := []Image{} + images := []*Image{} idlist := []string{} ids := make(map[string]*Image) names := make(map[string]*Image) if err = json.Unmarshal(data, &images); len(data) == 0 || err == nil { for n, image := range images { - ids[image.ID] = &images[n] + ids[image.ID] = images[n] idlist = append(idlist, image.ID) for _, name := range image.Names { if conflict, ok := names[name]; ok { r.removeName(conflict, name) - needSave = true + shouldSave = true } - names[name] = &images[n] + names[name] = images[n] } } } + if shouldSave && !r.IsReadWrite() { + return errors.New("image store assigns the same name to multiple images") + } r.images = images r.idindex = truncindex.NewTruncIndex(idlist) r.byid = ids r.byname = names - if needSave { - r.Touch() + if shouldSave { return r.Save() } return nil } func (r *imageStore) Save() error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify the image store at %q", r.imagespath()) + } rpath := r.imagespath() if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { return err @@ -154,6 +177,7 @@ func (r *imageStore) Save() error { if err != nil { return err } + defer r.Touch() return ioutils.AtomicWriteFile(rpath, jdata, 0600) } @@ -170,7 +194,27 @@ func newImageStore(dir string) (ImageStore, error) { istore := imageStore{ lockfile: lockfile, dir: dir, - images: []Image{}, + images: []*Image{}, + byid: make(map[string]*Image), + byname: make(map[string]*Image), + } + if err := istore.Load(); err != nil { + return nil, err + } + return &istore, nil +} + +func newROImageStore(dir string) (ROImageStore, error) { + lockfile, err := GetROLockfile(filepath.Join(dir, "images.lock")) + if err != nil { + return nil, err + } + lockfile.Lock() + defer lockfile.Unlock() + istore := imageStore{ + lockfile: lockfile, + dir: dir, + images: []*Image{}, byid: make(map[string]*Image), byname: make(map[string]*Image), } @@ -193,6 +237,9 @@ func (r *imageStore) lookup(id string) (*Image, bool) { } func (r *imageStore) ClearFlag(id string, flag string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to clear flags on images at %q", r.imagespath()) + } image, ok := r.lookup(id) if !ok { return ErrImageUnknown @@ -202,6 +249,9 @@ func (r *imageStore) ClearFlag(id string, flag string) error { } func (r *imageStore) SetFlag(id string, flag string, value interface{}) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to set flags on images at %q", r.imagespath()) + } image, ok := r.lookup(id) if !ok { return ErrImageUnknown @@ -210,7 +260,10 @@ func (r *imageStore) SetFlag(id string, flag string, value interface{}) error { return r.Save() } -func (r *imageStore) Create(id string, names []string, layer, metadata string) (image *Image, err error) { +func (r *imageStore) Create(id string, names []string, layer, metadata string, created time.Time) (image *Image, err error) { + if !r.IsReadWrite() { + return nil, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new images at %q", r.imagespath()) + } if id == "" { id = stringid.GenerateRandomID() _, idInUse := r.byid[id] @@ -227,18 +280,21 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string) ( return nil, ErrDuplicateName } } + if created.IsZero() { + created = time.Now().UTC() + } if err == nil { - newImage := Image{ + image = &Image{ ID: id, Names: names, TopLayer: layer, Metadata: metadata, BigDataNames: []string{}, BigDataSizes: make(map[string]int64), + Created: created, Flags: make(map[string]interface{}), } - r.images = append(r.images, newImage) - image = &r.images[len(r.images)-1] + r.images = append(r.images, image) r.idindex.Add(id) r.byid[id] = image for _, name := range names { @@ -257,6 +313,9 @@ func (r *imageStore) Metadata(id string) (string, error) { } func (r *imageStore) SetMetadata(id, metadata string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify image metadata at %q", r.imagespath()) + } if image, ok := r.lookup(id); ok { image.Metadata = metadata return r.Save() @@ -269,6 +328,9 @@ func (r *imageStore) removeName(image *Image, name string) { } func (r *imageStore) SetNames(id string, names []string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to change image name assignments at %q", r.imagespath()) + } if image, ok := r.lookup(id); ok { for _, name := range image.Names { delete(r.byname, name) @@ -286,15 +348,18 @@ func (r *imageStore) SetNames(id string, names []string) error { } func (r *imageStore) Delete(id string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to delete images at %q", r.imagespath()) + } image, ok := r.lookup(id) if !ok { return ErrImageUnknown } id = image.ID - newImages := []Image{} - for _, candidate := range r.images { - if candidate.ID != id { - newImages = append(newImages, candidate) + toDeleteIndex := -1 + for i, candidate := range r.images { + if candidate.ID == id { + toDeleteIndex = i } } delete(r.byid, id) @@ -302,7 +367,14 @@ func (r *imageStore) Delete(id string) error { for _, name := range image.Names { delete(r.byname, name) } - r.images = newImages + if toDeleteIndex != -1 { + // delete the image at toDeleteIndex + if toDeleteIndex == len(r.images)-1 { + r.images = r.images[:len(r.images)-1] + } else { + r.images = append(r.images[:toDeleteIndex], r.images[toDeleteIndex+1:]...) + } + } if err := r.Save(); err != nil { return err } @@ -359,6 +431,9 @@ func (r *imageStore) BigDataNames(id string) ([]string, error) { } func (r *imageStore) SetBigData(id, key string, data []byte) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to save data items associated with images at %q", r.imagespath()) + } image, ok := r.lookup(id) if !ok { return ErrImageUnknown @@ -393,6 +468,9 @@ func (r *imageStore) SetBigData(id, key string, data []byte) error { } func (r *imageStore) Wipe() error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to delete images at %q", r.imagespath()) + } ids := []string{} for id := range r.byid { ids = append(ids, id) @@ -421,6 +499,10 @@ func (r *imageStore) Modified() (bool, error) { return r.lockfile.Modified() } +func (r *imageStore) IsReadWrite() bool { + return r.lockfile.IsReadWrite() +} + func (r *imageStore) TouchedSince(when time.Time) bool { return r.lockfile.TouchedSince(when) } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 9a5aac6f..972d3a99 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -4,7 +4,6 @@ import ( "bytes" "compress/gzip" "encoding/json" - "errors" "io" "io/ioutil" "os" @@ -16,6 +15,8 @@ import ( "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/stringid" "github.com/containers/storage/pkg/truncindex" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) @@ -66,6 +67,38 @@ type Layer struct { // mounted at the mount point. MountCount int `json:"-"` + // Created is the datestamp for when this layer was created. Older + // versions of the library did not track this information, so callers + // will likely want to use the IsZero() method to verify that a value + // is set before using it. + Created time.Time `json:"created,omitempty"` + + // CompressedDigest is the digest of the blob that was last passed to + // ApplyDiff() or Put(), as it was presented to us. + CompressedDigest digest.Digest `json:"compressed-diff-digest,omitempty"` + + // CompressedSize is the length of the blob that was last passed to + // ApplyDiff() or Put(), as it was presented to us. If + // CompressedDigest is not set, this should be treated as if it were an + // uninitialized value. + CompressedSize int64 `json:"compressed-size,omitempty"` + + // UncompressedDigest is the digest of the blob that was last passed to + // ApplyDiff() or Put(), after we decompressed it. Often referred to + // as a DiffID. + UncompressedDigest digest.Digest `json:"diff-digest,omitempty"` + + // UncompressedSize is the length of the blob that was last passed to + // ApplyDiff() or Put(), after we decompressed it. If + // UncompressedDigest is not set, this should be treated as if it were + // an uninitialized value. + UncompressedSize int64 `json:"diff-size,omitempty"` + + // CompressionType is the type of compression which we detected on the blob + // that was last passed to ApplyDiff() or Put(). + CompressionType archive.Compression `json:"compression,omitempty"` + + // Flags is arbitrary data about the layer. Flags map[string]interface{} `json:"flags,omitempty"` } @@ -75,12 +108,74 @@ type layerMountPoint struct { MountCount int `json:"count"` } +// DiffOptions override the default behavior of Diff() methods. +type DiffOptions struct { + // Compression, if set overrides the default compressor when generating a diff. + Compression *archive.Compression +} + +// ROLayerStore wraps a graph driver, adding the ability to refer to layers by +// name, and keeping track of parent-child relationships, along with a list of +// all known layers. +type ROLayerStore interface { + ROFileBasedStore + ROMetadataStore + + // Exists checks if a layer with the specified name or ID is known. + Exists(id string) bool + + // Get retrieves information about a layer given an ID or name. + Get(id string) (*Layer, error) + + // Status returns an slice of key-value pairs, suitable for human consumption, + // relaying whatever status information the underlying driver can share. + Status() ([][2]string, error) + + // Changes returns a slice of Change structures, which contain a pathname + // (Path) and a description of what sort of change (Kind) was made by the + // layer (either ChangeModify, ChangeAdd, or ChangeDelete), relative to a + // specified layer. By default, the layer's parent is used as a reference. + Changes(from, to string) ([]archive.Change, error) + + // Diff produces a tarstream which can be applied to a layer with the contents + // of the first layer to produce a layer with the contents of the second layer. + // By default, the parent of the second layer is used as the first + // layer, so it need not be specified. Options can be used to override + // default behavior, but are also not required. + Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) + + // DiffSize produces an estimate of the length of the tarstream which would be + // produced by Diff. + DiffSize(from, to string) (int64, error) + + // Size produces a cached value for the uncompressed size of the layer, + // if one is known, or -1 if it is not known. If the layer can not be + // found, it returns an error. + Size(name string) (int64, error) + + // Lookup attempts to translate a name to an ID. Most methods do this + // implicitly. + Lookup(name string) (string, error) + + // LayersByCompressedDigest returns a slice of the layers with the + // specified compressed digest value recorded for them. + LayersByCompressedDigest(d digest.Digest) ([]Layer, error) + + // LayersByUncompressedDigest returns a slice of the layers with the + // specified uncompressed digest value recorded for them. + LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) + + // Layers returns a slice of the known layers. + Layers() ([]Layer, error) +} + // LayerStore wraps a graph driver, adding the ability to refer to layers by // name, and keeping track of parent-child relationships, along with a list of // all known layers. type LayerStore interface { - FileBasedStore - MetadataStore + ROLayerStore + RWFileBasedStore + RWMetadataStore FlaggableStore // Create creates a new layer, optionally giving it a specified ID rather than @@ -98,20 +193,10 @@ type LayerStore interface { // Put combines the functions of CreateWithFlags and ApplyDiff. Put(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}, diff archive.Reader) (*Layer, int64, error) - // Exists checks if a layer with the specified name or ID is known. - Exists(id string) bool - - // Get retrieves information about a layer given an ID or name. - Get(id string) (*Layer, error) - // SetNames replaces the list of names associated with a layer with the // supplied values. SetNames(id string, names []string) error - // Status returns an slice of key-value pairs, suitable for human consumption, - // relaying whatever status information the underlying driver can share. - Status() ([][2]string, error) - // Delete deletes a layer with the specified name or ID. Delete(id string) error @@ -126,48 +211,31 @@ type LayerStore interface { // Unmount unmounts a layer when it is no longer in use. Unmount(id string) error - // Changes returns a slice of Change structures, which contain a pathname - // (Path) and a description of what sort of change (Kind) was made by the - // layer (either ChangeModify, ChangeAdd, or ChangeDelete), relative to a - // specified layer. By default, the layer's parent is used as a reference. - Changes(from, to string) ([]archive.Change, error) - - // Diff produces a tarstream which can be applied to a layer with the contents - // of the first layer to produce a layer with the contents of the second layer. - // By default, the parent of the second layer is used as the first - // layer, so it need not be specified. - Diff(from, to string) (io.ReadCloser, error) - - // DiffSize produces an estimate of the length of the tarstream which would be - // produced by Diff. - DiffSize(from, to string) (int64, error) - // ApplyDiff reads a tarstream which was created by a previous call to Diff and // applies its changes to a specified layer. ApplyDiff(to string, diff archive.Reader) (int64, error) - - // Lookup attempts to translate a name to an ID. Most methods do this - // implicitly. - Lookup(name string) (string, error) - - // Layers returns a slice of the known layers. - Layers() ([]Layer, error) } type layerStore struct { - lockfile Locker - rundir string - driver drivers.Driver - layerdir string - layers []Layer - idindex *truncindex.TruncIndex - byid map[string]*Layer - byname map[string]*Layer - bymount map[string]*Layer + lockfile Locker + rundir string + driver drivers.Driver + layerdir string + layers []*Layer + idindex *truncindex.TruncIndex + byid map[string]*Layer + byname map[string]*Layer + bymount map[string]*Layer + bycompressedsum map[digest.Digest][]string + byuncompressedsum map[digest.Digest][]string } func (r *layerStore) Layers() ([]Layer, error) { - return r.layers, nil + layers := make([]Layer, len(r.layers)) + for i := range r.layers { + layers[i] = *(r.layers[i]) + } + return layers, nil } func (r *layerStore) mountspath() string { @@ -179,36 +247,41 @@ func (r *layerStore) layerspath() string { } func (r *layerStore) Load() error { - needSave := false + shouldSave := false rpath := r.layerspath() data, err := ioutil.ReadFile(rpath) if err != nil && !os.IsNotExist(err) { return err } - layers := []Layer{} + layers := []*Layer{} idlist := []string{} ids := make(map[string]*Layer) names := make(map[string]*Layer) mounts := make(map[string]*Layer) - parents := make(map[string][]*Layer) + compressedsums := make(map[digest.Digest][]string) + uncompressedsums := make(map[digest.Digest][]string) if err = json.Unmarshal(data, &layers); len(data) == 0 || err == nil { for n, layer := range layers { - ids[layer.ID] = &layers[n] + ids[layer.ID] = layers[n] idlist = append(idlist, layer.ID) for _, name := range layer.Names { if conflict, ok := names[name]; ok { r.removeName(conflict, name) - needSave = true + shouldSave = true } - names[name] = &layers[n] + names[name] = layers[n] } - if pslice, ok := parents[layer.Parent]; ok { - parents[layer.Parent] = append(pslice, &layers[n]) - } else { - parents[layer.Parent] = []*Layer{&layers[n]} + if layer.CompressedDigest != "" { + compressedsums[layer.CompressedDigest] = append(compressedsums[layer.CompressedDigest], layer.ID) + } + if layer.UncompressedDigest != "" { + uncompressedsums[layer.UncompressedDigest] = append(uncompressedsums[layer.UncompressedDigest], layer.ID) } } } + if shouldSave && !r.IsReadWrite() { + return errors.New("layer store assigns the same name to multiple layers") + } mpath := r.mountspath() data, err = ioutil.ReadFile(mpath) if err != nil && !os.IsNotExist(err) { @@ -231,29 +304,35 @@ func (r *layerStore) Load() error { r.byid = ids r.byname = names r.bymount = mounts + r.bycompressedsum = compressedsums + r.byuncompressedsum = uncompressedsums err = nil - // Last step: try to remove anything that a previous user of this - // storage area marked for deletion but didn't manage to actually - // delete. - for _, layer := range r.layers { - if cleanup, ok := layer.Flags[incompleteFlag]; ok { - if b, ok := cleanup.(bool); ok && b { - err = r.Delete(layer.ID) - if err != nil { - break + // Last step: if we're writable, try to remove anything that a previous + // user of this storage area marked for deletion but didn't manage to + // actually delete. + if r.IsReadWrite() { + for _, layer := range r.layers { + if cleanup, ok := layer.Flags[incompleteFlag]; ok { + if b, ok := cleanup.(bool); ok && b { + err = r.Delete(layer.ID) + if err != nil { + break + } + shouldSave = true } - needSave = true } } - } - if needSave { - r.Touch() - return r.Save() + if shouldSave { + return r.Save() + } } return err } func (r *layerStore) Save() error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify the layer store at %q", r.layerspath()) + } rpath := r.layerspath() if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { return err @@ -283,6 +362,7 @@ func (r *layerStore) Save() error { if err := ioutils.AtomicWriteFile(rpath, jldata, 0600); err != nil { return err } + defer r.Touch() return ioutils.AtomicWriteFile(mpath, jmdata, 0600) } @@ -314,6 +394,28 @@ func newLayerStore(rundir string, layerdir string, driver drivers.Driver) (Layer return &rlstore, nil } +func newROLayerStore(rundir string, layerdir string, driver drivers.Driver) (ROLayerStore, error) { + lockfile, err := GetROLockfile(filepath.Join(layerdir, "layers.lock")) + if err != nil { + return nil, err + } + lockfile.Lock() + defer lockfile.Unlock() + rlstore := layerStore{ + lockfile: lockfile, + driver: driver, + rundir: rundir, + layerdir: layerdir, + byid: make(map[string]*Layer), + bymount: make(map[string]*Layer), + byname: make(map[string]*Layer), + } + if err := rlstore.Load(); err != nil { + return nil, err + } + return &rlstore, nil +} + func (r *layerStore) lookup(id string) (*Layer, bool) { if layer, ok := r.byid[id]; ok { return layer, ok @@ -326,7 +428,24 @@ func (r *layerStore) lookup(id string) (*Layer, bool) { return nil, false } +func (r *layerStore) Size(name string) (int64, error) { + layer, ok := r.lookup(name) + if !ok { + return -1, ErrLayerUnknown + } + // We use the presence of a non-empty digest as an indicator that the size value was intentionally set, and that + // a zero value is not just present because it was never set to anything else (which can happen if the layer was + // created by a version of this library that didn't keep track of digest and size information). + if layer.UncompressedDigest != "" { + return layer.UncompressedSize, nil + } + return -1, nil +} + func (r *layerStore) ClearFlag(id string, flag string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to clear flags on layers at %q", r.layerspath()) + } layer, ok := r.lookup(id) if !ok { return ErrLayerUnknown @@ -336,6 +455,9 @@ func (r *layerStore) ClearFlag(id string, flag string) error { } func (r *layerStore) SetFlag(id string, flag string, value interface{}) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to set flags on layers at %q", r.layerspath()) + } layer, ok := r.lookup(id) if !ok { return ErrLayerUnknown @@ -349,6 +471,9 @@ func (r *layerStore) Status() ([][2]string, error) { } func (r *layerStore) Put(id, parent string, names []string, mountLabel string, options map[string]string, writeable bool, flags map[string]interface{}, diff archive.Reader) (layer *Layer, size int64, err error) { + if !r.IsReadWrite() { + return nil, -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new layers at %q", r.layerspath()) + } size = -1 if err := os.MkdirAll(r.rundir, 0700); err != nil { return nil, -1, err @@ -383,15 +508,15 @@ func (r *layerStore) Put(id, parent string, names []string, mountLabel string, o err = r.driver.Create(id, parent, mountLabel, options) } if err == nil { - newLayer := Layer{ + layer = &Layer{ ID: id, Parent: parent, Names: names, MountLabel: mountLabel, + Created: time.Now().UTC(), Flags: make(map[string]interface{}), } - r.layers = append(r.layers, newLayer) - layer = &r.layers[len(r.layers)-1] + r.layers = append(r.layers, layer) r.idindex.Add(id) r.byid[id] = layer for _, name := range names { @@ -441,6 +566,9 @@ func (r *layerStore) Create(id, parent string, names []string, mountLabel string } func (r *layerStore) Mount(id, mountLabel string) (string, error) { + if !r.IsReadWrite() { + return "", errors.Wrapf(ErrStoreIsReadOnly, "not allowed to update mount locations for layers at %q", r.mountspath()) + } layer, ok := r.lookup(id) if !ok { return "", ErrLayerUnknown @@ -466,6 +594,9 @@ func (r *layerStore) Mount(id, mountLabel string) (string, error) { } func (r *layerStore) Unmount(id string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to update mount locations for layers at %q", r.mountspath()) + } layer, ok := r.lookup(id) if !ok { layerByMount, ok := r.bymount[filepath.Clean(id)] @@ -495,6 +626,9 @@ func (r *layerStore) removeName(layer *Layer, name string) { } func (r *layerStore) SetNames(id string, names []string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to change layer name assignments at %q", r.layerspath()) + } if layer, ok := r.lookup(id); ok { for _, name := range layer.Names { delete(r.byname, name) @@ -519,6 +653,9 @@ func (r *layerStore) Metadata(id string) (string, error) { } func (r *layerStore) SetMetadata(id, metadata string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify layer metadata at %q", r.layerspath()) + } if layer, ok := r.lookup(id); ok { layer.Metadata = metadata return r.Save() @@ -531,6 +668,9 @@ func (r *layerStore) tspath(id string) string { } func (r *layerStore) Delete(id string) error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to delete layers at %q", r.layerspath()) + } layer, ok := r.lookup(id) if !ok { return ErrLayerUnknown @@ -549,13 +689,21 @@ func (r *layerStore) Delete(id string) error { if layer.MountPoint != "" { delete(r.bymount, layer.MountPoint) } - newLayers := []Layer{} - for _, candidate := range r.layers { - if candidate.ID != id { - newLayers = append(newLayers, candidate) + toDeleteIndex := -1 + for i, candidate := range r.layers { + if candidate.ID == id { + toDeleteIndex = i + break + } + } + if toDeleteIndex != -1 { + // delete the layer at toDeleteIndex + if toDeleteIndex == len(r.layers)-1 { + r.layers = r.layers[:len(r.layers)-1] + } else { + r.layers = append(r.layers[:toDeleteIndex], r.layers[toDeleteIndex+1:]...) } } - r.layers = newLayers if err = r.Save(); err != nil { return err } @@ -583,6 +731,9 @@ func (r *layerStore) Get(id string) (*Layer, error) { } func (r *layerStore) Wipe() error { + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to delete layers at %q", r.layerspath()) + } ids := []string{} for id := range r.byid { ids = append(ids, id) @@ -657,20 +808,20 @@ func (r *layerStore) newFileGetter(id string) (drivers.FileGetCloser, error) { }, nil } -func (r *layerStore) Diff(from, to string) (io.ReadCloser, error) { +func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) { var metadata storage.Unpacker from, to, toLayer, err := r.findParentAndLayer(from, to) if err != nil { return nil, ErrLayerUnknown } - compression := archive.Uncompressed - if cflag, ok := toLayer.Flags[compressionFlag]; ok { - if ctype, ok := cflag.(float64); ok { - compression = archive.Compression(ctype) - } else if ctype, ok := cflag.(archive.Compression); ok { - compression = archive.Compression(ctype) - } + // Default to applying the type of encryption that we noted was used + // for the layerdiff when it was applied. + compression := toLayer.CompressionType + // If a particular compression type (or no compression) was selected, + // use that instead. + if options != nil && options.Compression != nil { + compression = *options.Compression } if from != toLayer.Parent { diff, err := r.driver.Diff(to, from) @@ -758,6 +909,10 @@ func (r *layerStore) DiffSize(from, to string) (size int64, err error) { } func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err error) { + if !r.IsReadWrite() { + return -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify layer contents at %q", r.layerspath()) + } + layer, ok := r.lookup(to) if !ok { return -1, ErrLayerUnknown @@ -770,7 +925,9 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err } compression := archive.DetectCompression(header[:n]) - defragmented := io.MultiReader(bytes.NewBuffer(header[:n]), diff) + compressedDigest := digest.Canonical.Digester() + compressedCounter := ioutils.NewWriteCounter(compressedDigest.Hash()) + defragmented := io.TeeReader(io.MultiReader(bytes.NewBuffer(header[:n]), diff), compressedCounter) tsdata := bytes.Buffer{} compressor, err := gzip.NewWriterLevel(&tsdata, gzip.BestSpeed) @@ -778,15 +935,20 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err compressor = gzip.NewWriter(&tsdata) } metadata := storage.NewJSONPacker(compressor) - decompressed, err := archive.DecompressStream(defragmented) + uncompressed, err := archive.DecompressStream(defragmented) if err != nil { return -1, err } - payload, err := asm.NewInputTarStream(decompressed, metadata, storage.NewDiscardFilePutter()) + uncompressedDigest := digest.Canonical.Digester() + uncompressedCounter := ioutils.NewWriteCounter(uncompressedDigest.Hash()) + payload, err := asm.NewInputTarStream(io.TeeReader(uncompressed, uncompressedCounter), metadata, storage.NewDiscardFilePutter()) if err != nil { return -1, err } size, err = r.driver.ApplyDiff(layer.ID, layer.Parent, payload) + if err != nil { + return -1, err + } compressor.Close() if err == nil { if err := os.MkdirAll(filepath.Dir(r.tspath(layer.ID)), 0700); err != nil { @@ -797,15 +959,57 @@ func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err } } - if compression != archive.Uncompressed { - layer.Flags[compressionFlag] = compression - } else { - delete(layer.Flags, compressionFlag) + updateDigestMap := func(m *map[digest.Digest][]string, oldvalue, newvalue digest.Digest, id string) { + var newList []string + if oldvalue != "" { + for _, value := range (*m)[oldvalue] { + if value != id { + newList = append(newList, value) + } + } + if len(newList) > 0 { + (*m)[oldvalue] = newList + } else { + delete(*m, oldvalue) + } + } + if newvalue != "" { + (*m)[newvalue] = append((*m)[newvalue], id) + } } + updateDigestMap(&r.bycompressedsum, layer.CompressedDigest, compressedDigest.Digest(), layer.ID) + layer.CompressedDigest = compressedDigest.Digest() + layer.CompressedSize = compressedCounter.Count + updateDigestMap(&r.byuncompressedsum, layer.UncompressedDigest, uncompressedDigest.Digest(), layer.ID) + layer.UncompressedDigest = uncompressedDigest.Digest() + layer.UncompressedSize = uncompressedCounter.Count + layer.CompressionType = compression + + err = r.Save() return size, err } +func (r *layerStore) layersByDigestMap(m map[digest.Digest][]string, d digest.Digest) ([]Layer, error) { + var layers []Layer + for _, layerID := range m[d] { + layer, ok := r.lookup(layerID) + if !ok { + return nil, ErrLayerUnknown + } + layers = append(layers, *layer) + } + return layers, nil +} + +func (r *layerStore) LayersByCompressedDigest(d digest.Digest) ([]Layer, error) { + return r.layersByDigestMap(r.bycompressedsum, d) +} + +func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) { + return r.layersByDigestMap(r.byuncompressedsum, d) +} + func (r *layerStore) Lock() { r.lockfile.Lock() } @@ -822,6 +1026,10 @@ func (r *layerStore) Modified() (bool, error) { return r.lockfile.Modified() } +func (r *layerStore) IsReadWrite() bool { + return r.lockfile.IsReadWrite() +} + func (r *layerStore) TouchedSince(when time.Time) bool { return r.lockfile.TouchedSince(when) } diff --git a/vendor/github.com/containers/storage/lockfile.go b/vendor/github.com/containers/storage/lockfile.go index 33f5822f..6e09b526 100644 --- a/vendor/github.com/containers/storage/lockfile.go +++ b/vendor/github.com/containers/storage/lockfile.go @@ -1,14 +1,15 @@ package storage import ( + "fmt" "os" "path/filepath" "sync" "time" - "golang.org/x/sys/unix" - "github.com/containers/storage/pkg/stringid" + "github.com/pkg/errors" + "golang.org/x/sys/unix" ) // A Locker represents a file lock where the file is used to cache an @@ -27,43 +28,80 @@ type Locker interface { // TouchedSince() checks if the most recent writer modified the file (likely using Touch()) after the specified time. TouchedSince(when time.Time) bool + + // IsReadWrite() checks if the lock file is read-write + IsReadWrite() bool } type lockfile struct { - mu sync.Mutex - file string - fd uintptr - lw string + mu sync.Mutex + file string + fd uintptr + lw string + locktype int16 } var ( lockfiles map[string]*lockfile lockfilesLock sync.Mutex + // ErrLockReadOnly indicates that the caller only took a read-only lock, and is not allowed to write + ErrLockReadOnly = errors.New("lock is not a read-write lock") ) -// GetLockfile opens a lock file, creating it if necessary. The Locker object -// return will be returned unlocked. +// GetLockfile opens a read-write lock file, creating it if necessary. The +// Locker object it returns will be returned unlocked. func GetLockfile(path string) (Locker, error) { lockfilesLock.Lock() defer lockfilesLock.Unlock() if lockfiles == nil { lockfiles = make(map[string]*lockfile) } - if locker, ok := lockfiles[filepath.Clean(path)]; ok { + cleanPath := filepath.Clean(path) + if locker, ok := lockfiles[cleanPath]; ok { + if !locker.IsReadWrite() { + return nil, errors.Wrapf(ErrLockReadOnly, "lock %q is a read-only lock", cleanPath) + } return locker, nil } - fd, err := unix.Open(filepath.Clean(path), os.O_RDWR|os.O_CREATE, unix.S_IRUSR|unix.S_IWUSR) + fd, err := unix.Open(cleanPath, os.O_RDWR|os.O_CREATE, unix.S_IRUSR|unix.S_IWUSR) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error opening %q", cleanPath) } - locker := &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID()} + unix.CloseOnExec(fd) + locker := &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_WRLCK} lockfiles[filepath.Clean(path)] = locker return locker, nil } +// GetROLockfile opens a read-only lock file. The Locker object it returns +// will be returned unlocked. +func GetROLockfile(path string) (Locker, error) { + lockfilesLock.Lock() + defer lockfilesLock.Unlock() + if lockfiles == nil { + lockfiles = make(map[string]*lockfile) + } + cleanPath := filepath.Clean(path) + if locker, ok := lockfiles[cleanPath]; ok { + if locker.IsReadWrite() { + return nil, fmt.Errorf("lock %q is a read-write lock", cleanPath) + } + return locker, nil + } + fd, err := unix.Open(cleanPath, os.O_RDONLY, 0) + if err != nil { + return nil, errors.Wrapf(err, "error opening %q", cleanPath) + } + unix.CloseOnExec(fd) + locker := &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_RDLCK} + lockfiles[filepath.Clean(path)] = locker + return locker, nil +} + +// Lock locks the lock file func (l *lockfile) Lock() { lk := unix.Flock_t{ - Type: unix.F_WRLCK, + Type: l.locktype, Whence: int16(os.SEEK_SET), Start: 0, Len: 0, @@ -75,6 +113,7 @@ func (l *lockfile) Lock() { } } +// Unlock unlocks the lock file func (l *lockfile) Unlock() { lk := unix.Flock_t{ Type: unix.F_UNLCK, @@ -89,6 +128,7 @@ func (l *lockfile) Unlock() { l.mu.Unlock() } +// Touch updates the lock file with the UID of the user func (l *lockfile) Touch() error { l.lw = stringid.GenerateRandomID() id := []byte(l.lw) @@ -110,6 +150,7 @@ func (l *lockfile) Touch() error { return nil } +// Modified indicates if the lock file has been updated since the last time it was loaded func (l *lockfile) Modified() (bool, error) { id := []byte(l.lw) _, err := unix.Seek(int(l.fd), 0, os.SEEK_SET) @@ -128,6 +169,7 @@ func (l *lockfile) Modified() (bool, error) { return l.lw != lw, nil } +// TouchedSince indicates if the lock file has been touched since the specified time func (l *lockfile) TouchedSince(when time.Time) bool { st := unix.Stat_t{} err := unix.Fstat(int(l.fd), &st) @@ -137,3 +179,8 @@ func (l *lockfile) TouchedSince(when time.Time) bool { touched := time.Unix(statTMtimeUnix(st)) return when.Before(touched) } + +// IsRWLock indicates if the lock file is a read-write lock +func (l *lockfile) IsReadWrite() bool { + return (l.locktype == unix.F_WRLCK) +} diff --git a/vendor/github.com/containers/storage/storageversion/version_lib.go b/vendor/github.com/containers/storage/storageversion/version_lib.go deleted file mode 100644 index 34a531a9..00000000 --- a/vendor/github.com/containers/storage/storageversion/version_lib.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build !containersstorageautogen - -// Package storageversion is auto-generated at build-time -package storageversion - -// Default build-time variable for library-import. -// This file is overridden on build with build-time informations. -const ( - GitCommit string = "library-import" - Version string = "library-import" - BuildTime string = "library-import" - IAmStatic string = "library-import" -) diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index d297097f..ecf4e05e 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -2,7 +2,7 @@ package storage import ( "encoding/base64" - "errors" + "fmt" "io" "io/ioutil" "os" @@ -14,12 +14,14 @@ import ( // register all of the built-in drivers _ "github.com/containers/storage/drivers/register" + "github.com/BurntSushi/toml" drivers "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/stringid" - "github.com/containers/storage/storageversion" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" ) var ( @@ -51,43 +53,62 @@ var ( ErrIncompleteOptions = errors.New("missing necessary StoreOptions") // ErrSizeUnknown is returned when the caller asks for the size of a big data item, but the Store couldn't determine the answer. ErrSizeUnknown = errors.New("size is not known") + // ErrStoreIsReadOnly is returned when the caller makes a call to a read-only store that would require modifying its contents. + ErrStoreIsReadOnly = errors.New("called a write method on a read-only store") // DefaultStoreOptions is a reasonable default set of options. DefaultStoreOptions StoreOptions stores []*store storesLock sync.Mutex ) -// FileBasedStore wraps up the most common methods of the various types of file-based -// data stores that we implement. -type FileBasedStore interface { +// ROFileBasedStore wraps up the methods of the various types of file-based +// data stores that we implement which are needed for both read-only and +// read-write files. +type ROFileBasedStore interface { Locker // Load reloads the contents of the store from disk. It should be called // with the lock held. Load() error +} +// RWFileBasedStore wraps up the methods of various types of file-based data +// stores that we implement using read-write files. +type RWFileBasedStore interface { // Save saves the contents of the store to disk. It should be called with // the lock held, and Touch() should be called afterward before releasing the // lock. Save() error } -// MetadataStore wraps up methods for getting and setting metadata associated with IDs. -type MetadataStore interface { +// FileBasedStore wraps up the common methods of various types of file-based +// data stores that we implement. +type FileBasedStore interface { + ROFileBasedStore + RWFileBasedStore +} + +// ROMetadataStore wraps a method for reading metadata associated with an ID. +type ROMetadataStore interface { // Metadata reads metadata associated with an item with the specified ID. Metadata(id string) (string, error) +} +// RWMetadataStore wraps a method for setting metadata associated with an ID. +type RWMetadataStore interface { // SetMetadata updates the metadata associated with the item with the specified ID. SetMetadata(id, metadata string) error } -// A BigDataStore wraps up the most common methods of the various types of -// file-based lookaside stores that we implement. -type BigDataStore interface { - // SetBigData stores a (potentially large) piece of data associated with this - // ID. - SetBigData(id, key string, data []byte) error +// MetadataStore wraps up methods for getting and setting metadata associated with IDs. +type MetadataStore interface { + ROMetadataStore + RWMetadataStore +} +// An ROBigDataStore wraps up the read-only big-data related methods of the +// various types of file-based lookaside stores that we implement. +type ROBigDataStore interface { // BigData retrieves a (potentially large) piece of data associated with // this ID, if it has previously been set. BigData(id, key string) ([]byte, error) @@ -101,6 +122,21 @@ type BigDataStore interface { BigDataNames(id string) ([]string, error) } +// A RWBigDataStore wraps up the read-write big-data related methods of the +// various types of file-based lookaside stores that we implement. +type RWBigDataStore interface { + // SetBigData stores a (potentially large) piece of data associated with this + // ID. + SetBigData(id, key string, data []byte) error +} + +// A BigDataStore wraps up the most common big-data related methods of the +// various types of file-based lookaside stores that we implement. +type BigDataStore interface { + ROBigDataStore + RWBigDataStore +} + // A FlaggableStore can have flags set and cleared on items which it manages. type FlaggableStore interface { // ClearFlag removes a named flag from an item in the store. @@ -147,14 +183,22 @@ type Store interface { // by the Store. GraphDriver() (drivers.Driver, error) - // LayerStore obtains and returns a handle to the layer store object used by + // LayerStore obtains and returns a handle to the writeable layer store object used by // the Store. LayerStore() (LayerStore, error) - // ImageStore obtains and returns a handle to the image store object used by + // ROLayerStore obtains additional read/only layer store objects used by + // the Store. + ROLayerStores() ([]ROLayerStore, error) + + // ImageStore obtains and returns a handle to the writable image store object used by // the Store. ImageStore() (ImageStore, error) + // ROImageStores obtains additional read/only image store objects used by + // the Store. + ROImageStores() ([]ROImageStore, error) + // ContainerStore obtains and returns a handle to the container store object // used by the Store. ContainerStore() (ContainerStore, error) @@ -255,8 +299,8 @@ type Store interface { DiffSize(from, to string) (int64, error) // Diff returns the tarstream which would specify the changes returned by - // Changes. - Diff(from, to string) (io.ReadCloser, error) + // Changes. If options are passed in, they can override default behaviors. + Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) // ApplyDiff applies a tarstream to a layer. Information about the tarstream // is cached with the layer. Typically, a layer which is populated using a @@ -264,6 +308,18 @@ type Store interface { // before or after the diff is applied. ApplyDiff(to string, diff archive.Reader) (int64, error) + // LayersByCompressedDigest returns a slice of the layers with the + // specified compressed digest value recorded for them. + LayersByCompressedDigest(d digest.Digest) ([]Layer, error) + + // LayersByUncompressedDigest returns a slice of the layers with the + // specified uncompressed digest value recorded for them. + LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) + + // LayerSize returns a cached approximation of the layer's size, or -1 + // if we don't have a value on hand. + LayerSize(id string) (int64, error) + // Layers returns a list of the currently known layers. Layers() ([]Layer, error) @@ -379,6 +435,9 @@ type Store interface { // ImageOptions is used for passing options to a Store's CreateImage() method. type ImageOptions struct { + // CreationDate, if not zero, will override the default behavior of marking the image as having been + // created when CreateImage() was called, recording CreationDate instead. + CreationDate time.Time } // ContainerOptions is used for passing options to a Store's CreateContainer() method. @@ -396,7 +455,9 @@ type store struct { gidMap []idtools.IDMap graphDriver drivers.Driver layerStore LayerStore + roLayerStores []ROLayerStore imageStore ImageStore + roImageStores []ROImageStore containerStore ContainerStore } @@ -512,6 +573,9 @@ func (s *store) load() error { return err } s.layerStore = rls + if _, err := s.ROLayerStores(); err != nil { + return err + } gipath := filepath.Join(s.graphRoot, driverPrefix+"images") if err := os.MkdirAll(gipath, 0700); err != nil { @@ -522,6 +586,10 @@ func (s *store) load() error { return err } s.imageStore = ris + if _, err := s.ROImageStores(); err != nil { + return err + } + gcpath := filepath.Join(s.graphRoot, driverPrefix+"containers") if err := os.MkdirAll(gcpath, 0700); err != nil { return err @@ -594,6 +662,32 @@ func (s *store) LayerStore() (LayerStore, error) { return s.layerStore, nil } +func (s *store) ROLayerStores() ([]ROLayerStore, error) { + s.graphLock.Lock() + defer s.graphLock.Unlock() + if s.roLayerStores != nil { + return s.roLayerStores, nil + } + driver, err := s.getGraphDriver() + if err != nil { + return nil, err + } + driverPrefix := s.graphDriverName + "-" + rlpath := filepath.Join(s.runRoot, driverPrefix+"layers") + if err := os.MkdirAll(rlpath, 0700); err != nil { + return nil, err + } + for _, store := range driver.AdditionalImageStores() { + glpath := filepath.Join(store, driverPrefix+"layers") + rls, err := newROLayerStore(rlpath, glpath, driver) + if err != nil { + return nil, err + } + s.roLayerStores = append(s.roLayerStores, rls) + } + return s.roLayerStores, nil +} + func (s *store) ImageStore() (ImageStore, error) { if s.imageStore != nil { return s.imageStore, nil @@ -601,6 +695,26 @@ func (s *store) ImageStore() (ImageStore, error) { return nil, ErrLoadError } +func (s *store) ROImageStores() ([]ROImageStore, error) { + if len(s.roImageStores) != 0 { + return s.roImageStores, nil + } + driver, err := s.getGraphDriver() + if err != nil { + return nil, err + } + driverPrefix := s.graphDriverName + "-" + for _, store := range driver.AdditionalImageStores() { + gipath := filepath.Join(store, driverPrefix+"images") + ris, err := newROImageStore(gipath) + if err != nil { + return nil, err + } + s.roImageStores = append(s.roImageStores, ris) + } + return s.roImageStores, nil +} + func (s *store) ContainerStore() (ContainerStore, error) { if s.containerStore != nil { return s.containerStore, nil @@ -613,10 +727,6 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w if err != nil { return nil, -1, err } - ristore, err := s.ImageStore() - if err != nil { - return nil, -1, err - } rcstore, err := s.ContainerStore() if err != nil { return nil, -1, err @@ -624,19 +734,11 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w rlstore.Lock() defer rlstore.Unlock() - defer rlstore.Touch() if modified, err := rlstore.Modified(); modified || err != nil { rlstore.Load() } - ristore.Lock() - defer ristore.Unlock() - defer ristore.Touch() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() - defer rcstore.Touch() if modified, err := rcstore.Modified(); modified || err != nil { rcstore.Load() } @@ -668,49 +770,46 @@ func (s *store) CreateLayer(id, parent string, names []string, mountLabel string } func (s *store) CreateImage(id string, names []string, layer, metadata string, options *ImageOptions) (*Image, error) { - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } - rcstore, err := s.ContainerStore() - if err != nil { - return nil, err - } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - defer ristore.Touch() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } - rcstore.Lock() - defer rcstore.Unlock() - defer rcstore.Touch() - if modified, err := rcstore.Modified(); modified || err != nil { - rcstore.Load() - } if id == "" { id = stringid.GenerateRandomID() } - ilayer, err := rlstore.Get(layer) + rlstore, err := s.LayerStore() if err != nil { return nil, err } + stores, err := s.ROLayerStores() + if err != nil { + return nil, err + } + stores = append([]ROLayerStore{rlstore}, stores...) + var ilayer *Layer + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + ilayer, err = rlstore.Get(layer) + if err == nil { + break + } + } if ilayer == nil { return nil, ErrLayerUnknown } layer = ilayer.ID - return ristore.Create(id, names, layer, metadata) + + ristore, err := s.ImageStore() + if err != nil { + return nil, err + } + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + return ristore.Create(id, names, layer, metadata, options.CreationDate) } func (s *store) CreateContainer(id string, names []string, image, layer, metadata string, options *ContainerOptions) (*Container, error) { @@ -718,33 +817,11 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat if err != nil { return nil, err } - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } - rcstore, err := s.ContainerStore() - if err != nil { - return nil, err - } - rlstore.Lock() defer rlstore.Unlock() - defer rlstore.Touch() if modified, err := rlstore.Modified(); modified || err != nil { rlstore.Load() } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } - rcstore.Lock() - defer rcstore.Unlock() - defer rcstore.Touch() - if modified, err := rcstore.Modified(); modified || err != nil { - rcstore.Load() - } - if id == "" { id = stringid.GenerateRandomID() } @@ -752,10 +829,27 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat imageTopLayer := "" imageID := "" if image != "" { - cimage, err := ristore.Get(image) + ristore, err := s.ImageStore() if err != nil { return nil, err } + stores, err := s.ROImageStores() + if err != nil { + return nil, err + } + stores = append([]ROImageStore{ristore}, stores...) + var cimage *Image + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + cimage, err = ristore.Get(image) + if err == nil { + break + } + } if cimage == nil { return nil, ErrImageUnknown } @@ -767,6 +861,15 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat return nil, err } layer = clayer.ID + rcstore, err := s.ContainerStore() + if err != nil { + return nil, err + } + rcstore.Lock() + defer rcstore.Unlock() + if modified, err := rcstore.Modified(); modified || err != nil { + rcstore.Load() + } container, err := rcstore.Create(id, names, imageID, layer, metadata) if err != nil || container == nil { rlstore.Delete(layer) @@ -805,15 +908,12 @@ func (s *store) SetMetadata(id, metadata string) error { } if rlstore.Exists(id) { - defer rlstore.Touch() return rlstore.SetMetadata(id, metadata) } if ristore.Exists(id) { - defer ristore.Touch() return ristore.SetMetadata(id, metadata) } if rcstore.Exists(id) { - defer rcstore.Touch() return rcstore.SetMetadata(id, metadata) } return ErrNotAnID @@ -824,37 +924,51 @@ func (s *store) Metadata(id string) (string, error) { if err != nil { return "", err } - ristore, err := s.ImageStore() + stores, err := s.ROLayerStores() if err != nil { return "", err } + stores = append([]ROLayerStore{rlstore}, stores...) + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + if rlstore.Exists(id) { + return rlstore.Metadata(id) + } + } + + istore, err := s.ImageStore() + if err != nil { + return "", err + } + istores, err := s.ROImageStores() + if err != nil { + return "", err + } + istores = append([]ROImageStore{istore}, istores...) + for _, ristore := range istores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + if ristore.Exists(id) { + return ristore.Metadata(id) + } + } + rcstore, err := s.ContainerStore() if err != nil { return "", err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { rcstore.Load() } - - if rlstore.Exists(id) { - return rlstore.Metadata(id) - } - if ristore.Exists(id) { - return ristore.Metadata(id) - } if rcstore.Exists(id) { return rcstore.Metadata(id) } @@ -862,92 +976,84 @@ func (s *store) Metadata(id string) (string, error) { } func (s *store) ListImageBigData(id string) ([]string, error) { - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } ristore, err := s.ImageStore() if err != nil { return nil, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROImageStores() + if err != nil { + return nil, err } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() + stores = append([]ROImageStore{ristore}, stores...) + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + bigDataNames, err := ristore.BigDataNames(id) + if err == nil { + return bigDataNames, err + } } - - return ristore.BigDataNames(id) + return nil, ErrImageUnknown } func (s *store) ImageBigDataSize(id, key string) (int64, error) { - rlstore, err := s.LayerStore() - if err != nil { - return -1, err - } ristore, err := s.ImageStore() if err != nil { return -1, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROImageStores() + if err != nil { + return -1, err } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() + stores = append([]ROImageStore{ristore}, stores...) + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + size, err := ristore.BigDataSize(id, key) + if err == nil { + return size, nil + } } - - return ristore.BigDataSize(id, key) + return -1, ErrSizeUnknown } func (s *store) ImageBigData(id, key string) ([]byte, error) { - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } ristore, err := s.ImageStore() if err != nil { return nil, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROImageStores() + if err != nil { + return nil, err } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() + stores = append([]ROImageStore{ristore}, stores...) + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + data, err := ristore.BigData(id, key) + if err == nil { + return data, nil + } } - return ristore.BigData(id, key) + return nil, ErrImageUnknown } func (s *store) SetImageBigData(id, key string, data []byte) error { - rlstore, err := s.LayerStore() - if err != nil { - return err - } ristore, err := s.ImageStore() if err != nil { return err } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } ristore.Lock() defer ristore.Unlock() if modified, err := ristore.Modified(); modified || err != nil { @@ -958,29 +1064,11 @@ func (s *store) SetImageBigData(id, key string, data []byte) error { } func (s *store) ListContainerBigData(id string) ([]string, error) { - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } rcstore, err := s.ContainerStore() if err != nil { return nil, err } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { @@ -991,29 +1079,10 @@ func (s *store) ListContainerBigData(id string) ([]string, error) { } func (s *store) ContainerBigDataSize(id, key string) (int64, error) { - rlstore, err := s.LayerStore() - if err != nil { - return -1, err - } - ristore, err := s.ImageStore() - if err != nil { - return -1, err - } rcstore, err := s.ContainerStore() if err != nil { return -1, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { @@ -1024,29 +1093,10 @@ func (s *store) ContainerBigDataSize(id, key string) (int64, error) { } func (s *store) ContainerBigData(id, key string) ([]byte, error) { - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } rcstore, err := s.ContainerStore() if err != nil { return nil, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { @@ -1057,29 +1107,10 @@ func (s *store) ContainerBigData(id, key string) ([]byte, error) { } func (s *store) SetContainerBigData(id, key string, data []byte) error { - rlstore, err := s.LayerStore() - if err != nil { - return err - } - ristore, err := s.ImageStore() - if err != nil { - return err - } rcstore, err := s.ContainerStore() if err != nil { return err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { @@ -1090,74 +1121,63 @@ func (s *store) SetContainerBigData(id, key string, data []byte) error { } func (s *store) Exists(id string) bool { - rcstore, err := s.ContainerStore() + lstore, err := s.LayerStore() if err != nil { return false } + lstores, err := s.ROLayerStores() + if err != nil { + return false + } + lstores = append([]ROLayerStore{lstore}, lstores...) + for _, rlstore := range lstores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + if rlstore.Exists(id) { + return true + } + } + ristore, err := s.ImageStore() if err != nil { return false } - rlstore, err := s.LayerStore() + stores, err := s.ROImageStores() if err != nil { return false } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores = append([]ROImageStore{ristore}, stores...) + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + if ristore.Exists(id) { + return true + } } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() + + rcstore, err := s.ContainerStore() + if err != nil { + return false } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { rcstore.Load() } - if rcstore.Exists(id) { return true } - if ristore.Exists(id) { - return true - } - return rlstore.Exists(id) + + return false } func (s *store) SetNames(id string, names []string) error { - rcstore, err := s.ContainerStore() - if err != nil { - return err - } - ristore, err := s.ImageStore() - if err != nil { - return err - } - rlstore, err := s.LayerStore() - if err != nil { - return err - } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } - rcstore.Lock() - defer rcstore.Unlock() - if modified, err := rcstore.Modified(); modified || err != nil { - rcstore.Load() - } - deduped := []string{} seen := make(map[string]bool) for _, name := range names { @@ -1167,12 +1187,41 @@ func (s *store) SetNames(id string, names []string) error { } } + rlstore, err := s.LayerStore() + if err != nil { + return err + } + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } if rlstore.Exists(id) { return rlstore.SetNames(id, deduped) } + + ristore, err := s.ImageStore() + if err != nil { + return err + } + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } if ristore.Exists(id) { return ristore.SetNames(id, deduped) } + + rcstore, err := s.ContainerStore() + if err != nil { + return err + } + rcstore.Lock() + defer rcstore.Unlock() + if modified, err := rcstore.Modified(); modified || err != nil { + rcstore.Load() + } if rcstore.Exists(id) { return rcstore.SetNames(id, deduped) } @@ -1180,41 +1229,55 @@ func (s *store) SetNames(id string, names []string) error { } func (s *store) Names(id string) ([]string, error) { - rcstore, err := s.ContainerStore() - if err != nil { - return nil, err - } - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } rlstore, err := s.LayerStore() if err != nil { return nil, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROLayerStores() + if err != nil { + return nil, err } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() + stores = append([]ROLayerStore{rlstore}, stores...) + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + if l, err := rlstore.Get(id); l != nil && err == nil { + return l.Names, nil + } + } + + ristore, err := s.ImageStore() + if err != nil { + return nil, err + } + ristores, err := s.ROImageStores() + if err != nil { + return nil, err + } + ristores = append([]ROImageStore{ristore}, ristores...) + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + if i, err := ristore.Get(id); i != nil && err == nil { + return i.Names, nil + } + } + + rcstore, err := s.ContainerStore() + if err != nil { + return nil, err } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { rcstore.Load() } - - if l, err := rlstore.Get(id); l != nil && err == nil { - return l.Names, nil - } - if i, err := ristore.Get(id); i != nil && err == nil { - return i.Names, nil - } if c, err := rcstore.Get(id); c != nil && err == nil { return c.Names, nil } @@ -1294,8 +1357,6 @@ func (s *store) DeleteLayer(id string) error { } if rlstore.Exists(id) { - defer rlstore.Touch() - defer rcstore.Touch() if l, err := rlstore.Get(id); err != nil { id = l.ID } @@ -1314,7 +1375,7 @@ func (s *store) DeleteLayer(id string) error { } for _, image := range images { if image.TopLayer == id { - return ErrLayerUsedByImage + return errors.Wrapf(ErrLayerUsedByImage, "Layer %v used by image %v", id, image.ID) } } containers, err := rcstore.Containers() @@ -1323,7 +1384,7 @@ func (s *store) DeleteLayer(id string) error { } for _, container := range containers { if container.LayerID == id { - return ErrLayerUsedByContainer + return errors.Wrapf(ErrLayerUsedByContainer, "Layer %v used by container %v", id, container.ID) } } return rlstore.Delete(id) @@ -1367,8 +1428,6 @@ func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) return nil, err } id = image.ID - defer rlstore.Touch() - defer ristore.Touch() containers, err := rcstore.Containers() if err != nil { return nil, err @@ -1377,8 +1436,8 @@ func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) for _, container := range containers { aContainerByImage[container.ImageID] = container.ID } - if _, ok := aContainerByImage[id]; ok { - return nil, ErrImageUsedByContainer + if container, ok := aContainerByImage[id]; ok { + return nil, errors.Wrapf(ErrImageUsedByContainer, "Image used by %v", container) } images, err := ristore.Images() if err != nil { @@ -1482,8 +1541,6 @@ func (s *store) DeleteContainer(id string) error { } if rcstore.Exists(id) { - defer rlstore.Touch() - defer rcstore.Touch() if container, err := rcstore.Get(id); err == nil { if rlstore.Exists(container.LayerID) { if err = rlstore.Delete(container.LayerID); err != nil { @@ -1540,8 +1597,6 @@ func (s *store) Delete(id string) error { } if rcstore.Exists(id) { - defer rlstore.Touch() - defer rcstore.Touch() if container, err := rcstore.Get(id); err == nil { if rlstore.Exists(container.LayerID) { if err = rlstore.Delete(container.LayerID); err != nil { @@ -1565,11 +1620,9 @@ func (s *store) Delete(id string) error { } } if ristore.Exists(id) { - defer ristore.Touch() return ristore.Delete(id) } if rlstore.Exists(id) { - defer rlstore.Touch() return rlstore.Delete(id) } return ErrLayerUnknown @@ -1591,19 +1644,16 @@ func (s *store) Wipe() error { rlstore.Lock() defer rlstore.Unlock() - defer rlstore.Touch() if modified, err := rlstore.Modified(); modified || err != nil { rlstore.Load() } ristore.Lock() defer ristore.Unlock() - defer ristore.Touch() if modified, err := ristore.Modified(); modified || err != nil { ristore.Load() } rcstore.Lock() defer rcstore.Unlock() - defer rcstore.Touch() if modified, err := rcstore.Modified(); modified || err != nil { rcstore.Load() } @@ -1626,67 +1676,45 @@ func (s *store) Status() ([][2]string, error) { } func (s *store) Version() ([][2]string, error) { - return [][2]string{ - {"GitCommit", storageversion.GitCommit}, - {"Version", storageversion.Version}, - {"BuildTime", storageversion.BuildTime}, - }, nil + return [][2]string{}, nil } func (s *store) Mount(id, mountLabel string) (string, error) { - rcstore, err := s.ContainerStore() - if err != nil { - return "", err + if layerID, err := s.ContainerLayerID(id); err == nil { + id = layerID } rlstore, err := s.LayerStore() if err != nil { return "", err } - rlstore.Lock() defer rlstore.Unlock() - defer rlstore.Touch() if modified, err := rlstore.Modified(); modified || err != nil { rlstore.Load() } - rcstore.Lock() - defer rcstore.Unlock() - if modified, err := rcstore.Modified(); modified || err != nil { - rcstore.Load() + if rlstore.Exists(id) { + return rlstore.Mount(id, mountLabel) } - - if c, err := rcstore.Get(id); c != nil && err == nil { - id = c.LayerID - } - return rlstore.Mount(id, mountLabel) + return "", ErrLayerUnknown } func (s *store) Unmount(id string) error { - rcstore, err := s.ContainerStore() - if err != nil { - return err + if layerID, err := s.ContainerLayerID(id); err == nil { + id = layerID } rlstore, err := s.LayerStore() if err != nil { return err } - rlstore.Lock() defer rlstore.Unlock() - defer rlstore.Touch() if modified, err := rlstore.Modified(); modified || err != nil { rlstore.Load() } - rcstore.Lock() - defer rcstore.Unlock() - if modified, err := rcstore.Modified(); modified || err != nil { - rcstore.Load() + if rlstore.Exists(id) { + return rlstore.Unmount(id) } - - if c, err := rcstore.Get(id); c != nil && err == nil { - id = c.LayerID - } - return rlstore.Unmount(id) + return ErrLayerUnknown } func (s *store) Changes(from, to string) ([]archive.Change, error) { @@ -1694,14 +1722,22 @@ func (s *store) Changes(from, to string) ([]archive.Change, error) { if err != nil { return nil, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROLayerStores() + if err != nil { + return nil, err } - - return rlstore.Changes(from, to) + stores = append([]ROLayerStore{rlstore}, stores...) + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + if rlstore.Exists(to) { + return rlstore.Changes(from, to) + } + } + return nil, ErrLayerUnknown } func (s *store) DiffSize(from, to string) (int64, error) { @@ -1709,29 +1745,45 @@ func (s *store) DiffSize(from, to string) (int64, error) { if err != nil { return -1, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROLayerStores() + if err != nil { + return -1, err } - - return rlstore.DiffSize(from, to) + stores = append([]ROLayerStore{rlstore}, stores...) + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + if rlstore.Exists(to) { + return rlstore.DiffSize(from, to) + } + } + return -1, ErrLayerUnknown } -func (s *store) Diff(from, to string) (io.ReadCloser, error) { +func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) { rlstore, err := s.LayerStore() if err != nil { return nil, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROLayerStores() + if err != nil { + return nil, err } - - return rlstore.Diff(from, to) + stores = append([]ROLayerStore{rlstore}, stores...) + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + if rlstore.Exists(to) { + return rlstore.Diff(from, to, options) + } + } + return nil, ErrLayerUnknown } func (s *store) ApplyDiff(to string, diff archive.Reader) (int64, error) { @@ -1739,79 +1791,137 @@ func (s *store) ApplyDiff(to string, diff archive.Reader) (int64, error) { if err != nil { return -1, err } - rlstore.Lock() defer rlstore.Unlock() if modified, err := rlstore.Modified(); modified || err != nil { rlstore.Load() } + if rlstore.Exists(to) { + return rlstore.ApplyDiff(to, diff) + } + return -1, ErrLayerUnknown +} - return rlstore.ApplyDiff(to, diff) +func (s *store) layersByMappedDigest(m func(ROLayerStore, digest.Digest) ([]Layer, error), d digest.Digest) ([]Layer, error) { + var layers []Layer + rlstore, err := s.LayerStore() + if err != nil { + return nil, err + } + + stores, err := s.ROLayerStores() + if err != nil { + return nil, err + } + stores = append([]ROLayerStore{rlstore}, stores...) + + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + slayers, err := m(rlstore, d) + if err != nil { + return nil, err + } + layers = append(layers, slayers...) + } + return layers, nil +} + +func (s *store) LayersByCompressedDigest(d digest.Digest) ([]Layer, error) { + return s.layersByMappedDigest(func(r ROLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByCompressedDigest(d) }, d) +} + +func (s *store) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error) { + return s.layersByMappedDigest(func(r ROLayerStore, d digest.Digest) ([]Layer, error) { return r.LayersByUncompressedDigest(d) }, d) +} + +func (s *store) LayerSize(id string) (int64, error) { + lstore, err := s.LayerStore() + if err != nil { + return -1, err + } + lstores, err := s.ROLayerStores() + if err != nil { + return -1, err + } + lstores = append([]ROLayerStore{lstore}, lstores...) + for _, rlstore := range lstores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + if rlstore.Exists(id) { + return rlstore.Size(id) + } + } + return -1, ErrLayerUnknown } func (s *store) Layers() ([]Layer, error) { + var layers []Layer rlstore, err := s.LayerStore() if err != nil { return nil, err } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROLayerStores() + if err != nil { + return nil, err } + stores = append([]ROLayerStore{rlstore}, stores...) - return rlstore.Layers() + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + slayers, err := rlstore.Layers() + if err != nil { + return nil, err + } + layers = append(layers, slayers...) + } + return layers, nil } func (s *store) Images() ([]Image, error) { - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } + var images []Image ristore, err := s.ImageStore() if err != nil { return nil, err } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROImageStores() + if err != nil { + return nil, err } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() + stores = append([]ROImageStore{ristore}, stores...) + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + simages, err := ristore.Images() + if err != nil { + return nil, err + } + images = append(images, simages...) } - - return ristore.Images() + return images, nil } func (s *store) Containers() ([]Container, error) { - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } rcstore, err := s.ContainerStore() if err != nil { return nil, err } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { @@ -1827,13 +1937,24 @@ func (s *store) Layer(id string) (*Layer, error) { return nil, err } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() + stores, err := s.ROLayerStores() + if err != nil { + return nil, err } + stores = append([]ROLayerStore{rlstore}, stores...) - return rlstore.Get(id) + for _, rlstore := range stores { + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + rlstore.Load() + } + layer, err := rlstore.Get(id) + if err == nil { + return layer, nil + } + } + return nil, ErrLayerUnknown } func (s *store) Image(id string) (*Image, error) { @@ -1841,88 +1962,66 @@ func (s *store) Image(id string) (*Image, error) { if err != nil { return nil, err } - rlstore, err := s.LayerStore() + stores, err := s.ROImageStores() + if err != nil { + return nil, err + } + stores = append([]ROImageStore{ristore}, stores...) + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + image, err := ristore.Get(id) + if err == nil { + return image, nil + } + } + return nil, ErrImageUnknown +} + +func (s *store) ImagesByTopLayer(id string) ([]*Image, error) { + images := []*Image{} + layer, err := s.Layer(id) if err != nil { return nil, err } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } - - return ristore.Get(id) -} - -func (s *store) ImagesByTopLayer(id string) ([]*Image, error) { ristore, err := s.ImageStore() if err != nil { return nil, err } - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } - - layer, err := rlstore.Get(id) + stores, err := s.ROImageStores() if err != nil { return nil, err } - images := []*Image{} - imageList, err := ristore.Images() - if err != nil { - return nil, err - } - for _, image := range imageList { - if image.TopLayer == layer.ID { - images = append(images, &image) + stores = append([]ROImageStore{ristore}, stores...) + for _, ristore := range stores { + ristore.Lock() + defer ristore.Unlock() + if modified, err := ristore.Modified(); modified || err != nil { + ristore.Load() + } + imageList, err := ristore.Images() + if err != nil { + return nil, err + } + for _, image := range imageList { + if image.TopLayer == layer.ID { + images = append(images, &image) + } } } - return images, nil } func (s *store) Container(id string) (*Container, error) { - ristore, err := s.ImageStore() - if err != nil { - return nil, err - } - rlstore, err := s.LayerStore() - if err != nil { - return nil, err - } rcstore, err := s.ContainerStore() if err != nil { return nil, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { @@ -1932,12 +2031,25 @@ func (s *store) Container(id string) (*Container, error) { return rcstore.Get(id) } -func (s *store) ContainerByLayer(id string) (*Container, error) { - ristore, err := s.ImageStore() +func (s *store) ContainerLayerID(id string) (string, error) { + rcstore, err := s.ContainerStore() if err != nil { - return nil, err + return "", err } - rlstore, err := s.LayerStore() + rcstore.Lock() + defer rcstore.Unlock() + if modified, err := rcstore.Modified(); modified || err != nil { + rcstore.Load() + } + container, err := rcstore.Get(id) + if err != nil { + return "", err + } + return container.LayerID, nil +} + +func (s *store) ContainerByLayer(id string) (*Container, error) { + layer, err := s.Layer(id) if err != nil { return nil, err } @@ -1945,27 +2057,11 @@ func (s *store) ContainerByLayer(id string) (*Container, error) { if err != nil { return nil, err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { rcstore.Load() } - - layer, err := rlstore.Get(id) - if err != nil { - return nil, err - } containerList, err := rcstore.Containers() if err != nil { return nil, err @@ -1980,29 +2076,10 @@ func (s *store) ContainerByLayer(id string) (*Container, error) { } func (s *store) ContainerDirectory(id string) (string, error) { - rlstore, err := s.LayerStore() - if err != nil { - return "", err - } - ristore, err := s.ImageStore() - if err != nil { - return "", err - } rcstore, err := s.ContainerStore() if err != nil { return "", err } - - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { @@ -2023,29 +2100,11 @@ func (s *store) ContainerDirectory(id string) (string, error) { } func (s *store) ContainerRunDirectory(id string) (string, error) { - rlstore, err := s.LayerStore() - if err != nil { - return "", err - } - ristore, err := s.ImageStore() - if err != nil { - return "", err - } rcstore, err := s.ContainerStore() if err != nil { return "", err } - rlstore.Lock() - defer rlstore.Unlock() - if modified, err := rlstore.Modified(); modified || err != nil { - rlstore.Load() - } - ristore.Lock() - defer ristore.Unlock() - if modified, err := ristore.Modified(); modified || err != nil { - ristore.Load() - } rcstore.Lock() defer rcstore.Unlock() if modified, err := rcstore.Modified(); modified || err != nil { @@ -2145,7 +2204,7 @@ func (s *store) Shutdown(force bool) ([]string, error) { } } if len(mounted) > 0 && err == nil { - err = ErrLayerUsedByContainer + err = errors.Wrap(ErrLayerUsedByContainer, "A layer is mounted") } if err == nil { err = s.graphDriver.Cleanup() @@ -2187,11 +2246,64 @@ func stringSliceWithoutValue(slice []string, value string) []string { return modified } +const configFile = "/etc/containers/storage.conf" + +// OptionsConfig represents the "storage.options" TOML config table. +type OptionsConfig struct { + // AdditionalImagesStores is the location of additional read/only + // Image stores. Usually used to access Networked File System + // for shared image content + AdditionalImageStores []string `toml:"additionalimagestores"` +} + +// TOML-friendly explicit tables used for conversions. +type tomlConfig struct { + Storage struct { + Driver string `toml:"driver"` + RunRoot string `toml:"runroot"` + GraphRoot string `toml:"graphroot"` + Options struct{ OptionsConfig } `toml:"options"` + } `toml:"storage"` +} + func init() { DefaultStoreOptions.RunRoot = "/var/run/containers/storage" DefaultStoreOptions.GraphRoot = "/var/lib/containers/storage" - DefaultStoreOptions.GraphDriverName = os.Getenv("STORAGE_DRIVER") - DefaultStoreOptions.GraphDriverOptions = strings.Split(os.Getenv("STORAGE_OPTS"), ",") + DefaultStoreOptions.GraphDriverName = "overlay" + + data, err := ioutil.ReadFile(configFile) + if err != nil { + if !os.IsNotExist(err) { + fmt.Printf("Failed to read %s %v\n", configFile, err.Error()) + return + } + } + + config := new(tomlConfig) + + if _, err := toml.Decode(string(data), config); err != nil { + fmt.Printf("Failed to parse %s %v\n", configFile, err.Error()) + return + } + if config.Storage.Driver != "" { + DefaultStoreOptions.GraphDriverName = config.Storage.Driver + } + if config.Storage.RunRoot != "" { + DefaultStoreOptions.RunRoot = config.Storage.RunRoot + } + if config.Storage.GraphRoot != "" { + DefaultStoreOptions.GraphRoot = config.Storage.GraphRoot + } + for _, s := range config.Storage.Options.AdditionalImageStores { + DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.imagestore=%s", config.Storage.Driver, s)) + } + + if os.Getenv("STORAGE_DRIVER") != "" { + DefaultStoreOptions.GraphDriverName = os.Getenv("STORAGE_DRIVER") + } + if os.Getenv("STORAGE_OPTS") != "" { + DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, strings.Split(os.Getenv("STORAGE_OPTS"), ",")...) + } if len(DefaultStoreOptions.GraphDriverOptions) == 1 && DefaultStoreOptions.GraphDriverOptions[0] == "" { DefaultStoreOptions.GraphDriverOptions = nil } diff --git a/vendor/github.com/containers/storage/vendor.conf b/vendor/github.com/containers/storage/vendor.conf index c7070f44..0f3adb02 100644 --- a/vendor/github.com/containers/storage/vendor.conf +++ b/vendor/github.com/containers/storage/vendor.conf @@ -1,3 +1,4 @@ +github.com/BurntSushi/toml master github.com/Microsoft/go-winio 307e919c663683a9000576fdc855acaf9534c165 github.com/Microsoft/hcsshim 0f615c198a84e0344b4ed49c464d8833d4648dfc github.com/Sirupsen/logrus 61e43dc76f7ee59a82bdf3d71033dc12bea4c77d @@ -7,9 +8,11 @@ github.com/docker/go-units 0dadbb0345b35ec7ef35e228dabb8de89a65bf52 github.com/go-check/check 20d25e2804050c1cd24a7eea1e7a6447dd0e74ec github.com/mattn/go-shellwords 753a2322a99f87c0eff284980e77f53041555bc6 github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 +github.com/opencontainers/go-digest master github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07 github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 +github.com/pkg/errors master github.com/tchap/go-patricia v2.2.6 github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721 github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3