Vendor in latest container/storage

This container/storage will mount /run/containers/storage as a
private mount so that cri-o stops leaking shm mountpoints in to the
hosts namespace.  Will also fix issues with oci-umount cleaning up
mount points.

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

cmd/crio/main.go

@@ -360,7 +360,16 @@ func main() {
 
 		return nil
 	}
-
+	app.After = func(*cli.Context) error {
+		// called by Run() when the command handler succeeds
+		libkpod.ShutdownStores(false)
+		return nil
+	}
+	cli.OsExiter = func(code int) {
+		// called by Run() when the command fails, bypassing After()
+		libkpod.ShutdownStores(false)
+		os.Exit(code)
+	}
 	app.Action = func(c *cli.Context) error {
 		if c.GlobalBool("profile") {
 			profilePort := c.GlobalInt("profile-port")

cmd/kpod/common.go

@@ -14,10 +14,6 @@ import (
 	"github.com/urfave/cli"
 )
 
-var (
-	stores = make(map[storage.Store]struct{})
-)
-
 func getStore(c *libkpod.Config) (storage.Store, error) {
 	options := storage.DefaultStoreOptions
 	options.GraphRoot = c.Root
@@ -30,7 +26,7 @@ func getStore(c *libkpod.Config) (storage.Store, error) {
 		return nil, err
 	}
 	is.Transport.SetStore(store)
-	stores[store] = struct{}{}
+	libkpod.AddStore(store)
 	return store, nil
 }
 

cmd/kpod/main.go

@@ -4,6 +4,7 @@ import (
 	"os"
 
 	"github.com/containers/storage/pkg/reexec"
+	"github.com/kubernetes-incubator/cri-o/libkpod"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -49,12 +50,12 @@ func main() {
 	}
 	app.After = func(*cli.Context) error {
 		// called by Run() when the command handler succeeds
-		shutdownStores()
+		libkpod.ShutdownStores(false)
 		return nil
 	}
 	cli.OsExiter = func(code int) {
 		// called by Run() when the command fails, bypassing After()
-		shutdownStores()
+		libkpod.ShutdownStores(false)
 		os.Exit(code)
 	}
 	app.Flags = []cli.Flag{

libkpod/container_server.go

@@ -99,6 +99,25 @@ func (c *ContainerServer) StorageRuntimeServer() storage.RuntimeServer {
 	return c.storageRuntimeServer
 }
 
+var (
+	stores = make(map[cstorage.Store]struct{})
+)
+
+// AddStore adds a store to the list of stores to shutdown, when exiting
+func AddStore(store cstorage.Store) {
+	stores[store] = struct{}{}
+}
+
+// ShutdownStores calls the Shutdown interface for all allocated stores,
+// cleaning up allocated resources.
+func ShutdownStores(force bool) {
+	for store := range stores {
+		if _, err := store.Shutdown(force); err != nil {
+			break
+		}
+	}
+}
+
 // New creates a new ContainerServer with options provided
 func New(config *Config) (*ContainerServer, error) {
 	store, err := cstorage.GetStore(cstorage.StoreOptions{
@@ -110,6 +129,7 @@ func New(config *Config) (*ContainerServer, error) {
 	if err != nil {
 		return nil, err
 	}
+	AddStore(store)
 
 	imageService, err := storage.GetImageService(store, config.DefaultTransport, config.InsecureRegistries, config.Registries)
 	if err != nil {

server/container_create.go

@@ -103,7 +103,7 @@ func addImageVolumes(rootfs string, s *Server, containerInfo *storage.ContainerI
 			}
 		case libkpod.ImageVolumesBind:
 			volumeDirName := stringid.GenerateNonCryptoID()
-			src := filepath.Join(containerInfo.RunDir, "mounts", volumeDirName)
+			src := filepath.Join(containerInfo.Dir, "mounts", volumeDirName)
 			if err1 := os.MkdirAll(src, 0644); err1 != nil {
 				return err1
 			}
@@ -975,7 +975,7 @@ func (s *Server) createSandboxContainer(ctx context.Context, containerID string,
 	if err = specgen.SaveToFile(filepath.Join(containerInfo.Dir, "config.json"), saveOptions); err != nil {
 		return nil, err
 	}
-	if err = specgen.SaveToFile(filepath.Join(containerInfo.RunDir, "config.json"), saveOptions); err != nil {
+	if err = specgen.SaveToFile(filepath.Join(containerInfo.Dir, "config.json"), saveOptions); err != nil {
 		return nil, err
 	}
 

server/sandbox_run.go

@@ -199,7 +199,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 		dnsServers := req.GetConfig().GetDnsConfig().Servers
 		dnsSearches := req.GetConfig().GetDnsConfig().Searches
 		dnsOptions := req.GetConfig().GetDnsConfig().Options
-		resolvPath = fmt.Sprintf("%s/resolv.conf", podContainer.RunDir)
+		resolvPath = fmt.Sprintf("%s/resolv.conf", podContainer.Dir)
 		err = parseDNSOptions(dnsServers, dnsSearches, dnsOptions, resolvPath)
 		if err != nil {
 			err1 := removeFile(resolvPath)
@@ -264,7 +264,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 	if req.GetConfig().GetLinux().GetSecurityContext().GetNamespaceOptions().HostIpc {
 		shmPath = "/dev/shm"
 	} else {
-		shmPath, err = setupShm(podContainer.RunDir, mountLabel)
+		shmPath, err = setupShm(podContainer.Dir, mountLabel)
 		if err != nil {
 			return nil, err
 		}
@@ -464,7 +464,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 	g.AddAnnotation(annotations.MountPoint, mountPoint)
 	g.SetRootPath(mountPoint)
 
-	container, err := oci.NewContainer(id, containerName, podContainer.RunDir, logPath, sb.NetNs(), labels, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
+	container, err := oci.NewContainer(id, containerName, podContainer.Dir, logPath, sb.NetNs(), labels, kubeAnnotations, "", "", "", nil, id, false, false, false, sb.Privileged(), sb.Trusted(), podContainer.Dir, created, podContainer.Config.Config.StopSignal)
 	if err != nil {
 		return nil, err
 	}
@@ -511,7 +511,7 @@ func (s *Server) RunPodSandbox(ctx context.Context, req *pb.RunPodSandboxRequest
 	if err != nil {
 		return nil, fmt.Errorf("failed to save template configuration for pod sandbox %s(%s): %v", sb.Name(), id, err)
 	}
-	if err = g.SaveToFile(filepath.Join(podContainer.RunDir, "config.json"), saveOptions); err != nil {
+	if err = g.SaveToFile(filepath.Join(podContainer.Dir, "config.json"), saveOptions); err != nil {
 		return nil, fmt.Errorf("failed to write runtime configuration for pod sandbox %s(%s): %v", sb.Name(), id, err)
 	}
 
@@ -580,8 +580,8 @@ func getSELinuxLabels(selinuxOptions *pb.SELinuxOption) (processLabel string, mo
 	return label.InitLabels(label.DupSecOpt(processLabel))
 }
 
-func setupShm(podSandboxRunDir, mountLabel string) (shmPath string, err error) {
+func setupShm(podSandboxDir, mountLabel string) (shmPath string, err error) {
-	shmPath = filepath.Join(podSandboxRunDir, "shm")
+	shmPath = filepath.Join(podSandboxDir, "shm")
 	if err = os.Mkdir(shmPath, 0700); err != nil {
 		return "", err
 	}

vendor.conf

@@ -8,7 +8,7 @@ github.com/sirupsen/logrus v1.0.0
 github.com/containers/image abb4cd79e3427bb2b02a5930814ef2ad19983c24
 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1
 github.com/ostreedev/ostree-go master
-github.com/containers/storage f8cff0727cf0802f0752ca58d2c05ec5270a47d5
+github.com/containers/storage 71b14b06d4f19e3870c24cc1bca0ede685a532c2 https://github.com/rhatdan/storage
 github.com/containernetworking/cni v0.4.0
 google.golang.org/grpc v1.0.4 https://github.com/grpc/grpc-go
 github.com/opencontainers/selinux v1.0.0-rc1

vendor/github.com/containers/storage/containers.go

@@ -2,7 +2,6 @@ package storage
 
 import (
 	"encoding/json"
-	"errors"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -13,11 +12,6 @@ import (
 	"github.com/containers/storage/pkg/truncindex"
 )
 
-var (
-	// ErrContainerUnknown indicates that there was no container with the specified name or ID
-	ErrContainerUnknown = errors.New("container not known")
-)
-
 // A Container is a reference to a read-write layer with metadata.
 type Container struct {
 	// ID is either one which was specified at create-time, or a random

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -96,7 +96,7 @@ func init() {
 // InitWithName returns the a naive diff driver for the overlay filesystem,
 // which returns the passed-in name when asked which driver it is.
 func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
-	opts, err := parseOptions(options)
+	opts, err := parseOptions(name, options)
 	if err != nil {
 		return nil, err
 	}
@@ -176,7 +176,7 @@ type overlayOptions struct {
 	imageStores         []string
 }
 
-func parseOptions(options []string) (*overlayOptions, error) {
+func parseOptions(name string, options []string) (*overlayOptions, error) {
 	o := &overlayOptions{}
 	for _, option := range options {
 		key, val, err := parsers.ParseKeyValueOpt(option)
@@ -190,24 +190,24 @@ func parseOptions(options []string) (*overlayOptions, error) {
 			if err != nil {
 				return nil, err
 			}
-		case "overlay.imagestore":
+		case "overlay.imagestore", "overlay2.imagestore":
 			// Additional read only image stores to use for lower paths
 			for _, store := range strings.Split(val, ",") {
 				store = filepath.Clean(store)
 				if !filepath.IsAbs(store) {
-					return nil, fmt.Errorf("overlay: image path %q is not absolute.  Can not be relative", store)
+					return nil, fmt.Errorf("%s: image path %q is not absolute.  Can not be relative", name, store)
 				}
 				st, err := os.Stat(store)
 				if err != nil {
-					return nil, fmt.Errorf("overlay: Can't stat imageStore dir %s: %v", store, err)
+					return nil, fmt.Errorf("%s: Can't stat imageStore dir %s: %v", name, store, err)
 				}
 				if !st.IsDir() {
-					return nil, fmt.Errorf("overlay: image path %q must be a directory", store)
+					return nil, fmt.Errorf("%s: image path %q must be a directory", name, store)
 				}
 				o.imageStores = append(o.imageStores, store)
 			}
 		default:
-			return nil, fmt.Errorf("overlay: Unknown option %s", key)
+			return nil, fmt.Errorf("%s: Unknown option %s", name, key)
 		}
 	}
 	return o, nil
@@ -516,7 +516,7 @@ func (d *Driver) Put(id string) error {
 			// We didn't have a "lower" directory, so we weren't mounting a "merged" directory anyway
 			return nil
 		}
-		logrus.Debugf("Failed to unmount %s overlay: %v", id, err)
+		logrus.Debugf("Failed to unmount %s %s: %v", id, d.name, err)
 	}
 	return err
 }

vendor/github.com/containers/storage/errors.go

@@ -0,0 +1,52 @@
+package storage
+
+import (
+	"errors"
+)
+
+var (
+	// ErrContainerUnknown indicates that there was no container with the specified name or ID.
+	ErrContainerUnknown = errors.New("container not known")
+	// ErrImageUnknown indicates that there was no image with the specified name or ID.
+	ErrImageUnknown = errors.New("image not known")
+	// ErrParentUnknown indicates that we didn't record the ID of the parent of the specified layer.
+	ErrParentUnknown = errors.New("parent of layer not known")
+	// ErrLayerUnknown indicates that there was no layer with the specified name or ID.
+	ErrLayerUnknown = errors.New("layer not known")
+	// ErrLoadError indicates that there was an initialization error.
+	ErrLoadError = errors.New("error loading storage metadata")
+	// ErrDuplicateID indicates that an ID which is to be assigned to a new item is already being used.
+	ErrDuplicateID = errors.New("that ID is already in use")
+	// ErrDuplicateName indicates that a name which is to be assigned to a new item is already being used.
+	ErrDuplicateName = errors.New("that name is already in use")
+	// ErrParentIsContainer is returned when a caller attempts to create a layer as a child of a container's layer.
+	ErrParentIsContainer = errors.New("would-be parent layer is a container")
+	// ErrNotAContainer is returned when the caller attempts to delete a container that isn't a container.
+	ErrNotAContainer = errors.New("identifier is not a container")
+	// ErrNotAnImage is returned when the caller attempts to delete an image that isn't an image.
+	ErrNotAnImage = errors.New("identifier is not an image")
+	// ErrNotALayer is returned when the caller attempts to delete a layer that isn't a layer.
+	ErrNotALayer = errors.New("identifier is not a layer")
+	// ErrNotAnID is returned when the caller attempts to read or write metadata from an item that doesn't exist.
+	ErrNotAnID = errors.New("identifier is not a layer, image, or container")
+	// ErrLayerHasChildren is returned when the caller attempts to delete a layer that has children.
+	ErrLayerHasChildren = errors.New("layer has children")
+	// ErrLayerUsedByImage is returned when the caller attempts to delete a layer that is an image's top layer.
+	ErrLayerUsedByImage = errors.New("layer is in use by an image")
+	// ErrLayerUsedByContainer is returned when the caller attempts to delete a layer that is a container's layer.
+	ErrLayerUsedByContainer = errors.New("layer is in use by a container")
+	// ErrImageUsedByContainer is returned when the caller attempts to delete an image that is a container's image.
+	ErrImageUsedByContainer = errors.New("image is in use by a container")
+	// ErrIncompleteOptions is returned when the caller attempts to initialize a Store without providing required information.
+	ErrIncompleteOptions = errors.New("missing necessary StoreOptions")
+	// ErrSizeUnknown is returned when the caller asks for the size of a big data item, but the Store couldn't determine the answer.
+	ErrSizeUnknown = errors.New("size is not known")
+	// ErrStoreIsReadOnly is returned when the caller makes a call to a read-only store that would require modifying its contents.
+	ErrStoreIsReadOnly = errors.New("called a write method on a read-only store")
+	// ErrLockReadOnly indicates that the caller only took a read-only lock, and is not allowed to write.
+	ErrLockReadOnly = errors.New("lock is not a read-write lock")
+	// ErrDuplicateImageNames indicates that the read-only store uses the same name for multiple images.
+	ErrDuplicateImageNames = errors.New("read-only image store assigns the same name to multiple images")
+	// ErrDuplicateLayerNames indicates that the read-only store uses the same name for multiple layers.
+	ErrDuplicateLayerNames = errors.New("read-only layer store assigns the same name to multiple layers")
+)

vendor/github.com/containers/storage/images.go

@@ -13,11 +13,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-var (
-	// ErrImageUnknown indicates that there was no image with the specified name or ID
-	ErrImageUnknown = errors.New("image not known")
-)
-
 // An Image is a reference to a layer and an associated metadata string.
 type Image struct {
 	// ID is either one which was specified at create-time, or a random
@@ -153,7 +148,7 @@ func (r *imageStore) Load() error {
 		}
 	}
 	if shouldSave && !r.IsReadWrite() {
-		return errors.New("image store assigns the same name to multiple images")
+		return ErrDuplicateImageNames
 	}
 	r.images = images
 	r.idindex = truncindex.NewTruncIndex(idlist)

vendor/github.com/containers/storage/layers.go

@@ -27,13 +27,6 @@ const (
 	compressionFlag = "diff-compression"
 )
 
-var (
-	// ErrParentUnknown indicates that we didn't record the ID of the parent of the specified layer
-	ErrParentUnknown = errors.New("parent of layer not known")
-	// ErrLayerUnknown indicates that there was no layer with the specified name or ID
-	ErrLayerUnknown = errors.New("layer not known")
-)
-
 // A Layer is a record of a copy-on-write layer that's stored by the lower
 // level graph driver.
 type Layer struct {
@@ -280,7 +273,7 @@ func (r *layerStore) Load() error {
 		}
 	}
 	if shouldSave && !r.IsReadWrite() {
-		return errors.New("layer store assigns the same name to multiple layers")
+		return ErrDuplicateLayerNames
 	}
 	mpath := r.mountspath()
 	data, err = ioutil.ReadFile(mpath)

vendor/github.com/containers/storage/lockfile.go

@@ -44,8 +44,6 @@ type lockfile struct {
 var (
 	lockfiles     map[string]*lockfile
 	lockfilesLock sync.Mutex
-	// ErrLockReadOnly indicates that the caller only took a read-only lock, and is not allowed to write
-	ErrLockReadOnly = errors.New("lock is not a read-write lock")
 )
 
 // GetLockfile opens a read-write lock file, creating it if necessary.  The

vendor/github.com/containers/storage/pkg/archive/example_changes.go

@@ -0,0 +1,97 @@
+// +build ignore
+
+// Simple tool to create an archive stream from an old and new directory
+//
+// By default it will stream the comparison of two temporary directories with junk files
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/containers/storage/pkg/archive"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	flDebug  = flag.Bool("D", false, "debugging output")
+	flNewDir = flag.String("newdir", "", "")
+	flOldDir = flag.String("olddir", "", "")
+	log      = logrus.New()
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)")
+		fmt.Printf("%s [OPTIONS]\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+	log.Out = os.Stderr
+	if (len(os.Getenv("DEBUG")) > 0) || *flDebug {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+	var newDir, oldDir string
+
+	if len(*flNewDir) == 0 {
+		var err error
+		newDir, err = ioutil.TempDir("", "storage-test-newDir")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(newDir)
+		if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		newDir = *flNewDir
+	}
+
+	if len(*flOldDir) == 0 {
+		oldDir, err := ioutil.TempDir("", "storage-test-oldDir")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(oldDir)
+	} else {
+		oldDir = *flOldDir
+	}
+
+	changes, err := archive.ChangesDirs(newDir, oldDir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	a, err := archive.ExportChanges(newDir, changes)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer a.Close()
+
+	i, err := io.Copy(os.Stdout, a)
+	if err != nil && err != io.EOF {
+		log.Fatal(err)
+	}
+	fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i)
+}
+
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}

vendor/github.com/containers/storage/pkg/mount/mount.go

@@ -2,6 +2,8 @@ package mount
 
 import (
 	"time"
+
+	"github.com/containers/storage/pkg/fileutils"
 )
 
 // GetMounts retrieves a list of mounts for the current running process.
@@ -17,6 +19,10 @@ func Mounted(mountpoint string) (bool, error) {
 		return false, err
 	}
 
+	mountpoint, err = fileutils.ReadSymlinkedDirectory(mountpoint)
+	if err != nil {
+		return false, err
+	}
 	// Search the table for the mountpoint
 	for _, e := range entries {
 		if e.Mountpoint == mountpoint {

vendor/github.com/containers/storage/pkg/mount/sharedsubtree_notlinux.go

@@ -0,0 +1,9 @@
+// +build !linux
+
+package mount
+
+// MakePrivate ensures a mounted filesystem has the PRIVATE mount option enabled.
+// See the supported options in flags.go for further reference.
+func MakePrivate(mountPoint string) error {
+	return nil
+}

vendor/github.com/containers/storage/store.go

@@ -25,36 +25,6 @@ import (
 )
 
 var (
-	// ErrLoadError indicates that there was an initialization error.
-	ErrLoadError = errors.New("error loading storage metadata")
-	// ErrDuplicateID indicates that an ID which is to be assigned to a new item is already being used.
-	ErrDuplicateID = errors.New("that ID is already in use")
-	// ErrDuplicateName indicates that a name which is to be assigned to a new item is already being used.
-	ErrDuplicateName = errors.New("that name is already in use")
-	// ErrParentIsContainer is returned when a caller attempts to create a layer as a child of a container's layer.
-	ErrParentIsContainer = errors.New("would-be parent layer is a container")
-	// ErrNotAContainer is returned when the caller attempts to delete a container that isn't a container.
-	ErrNotAContainer = errors.New("identifier is not a container")
-	// ErrNotAnImage is returned when the caller attempts to delete an image that isn't an image.
-	ErrNotAnImage = errors.New("identifier is not an image")
-	// ErrNotALayer is returned when the caller attempts to delete a layer that isn't a layer.
-	ErrNotALayer = errors.New("identifier is not a layer")
-	// ErrNotAnID is returned when the caller attempts to read or write metadata from an item that doesn't exist.
-	ErrNotAnID = errors.New("identifier is not a layer, image, or container")
-	// ErrLayerHasChildren is returned when the caller attempts to delete a layer that has children.
-	ErrLayerHasChildren = errors.New("layer has children")
-	// ErrLayerUsedByImage is returned when the caller attempts to delete a layer that is an image's top layer.
-	ErrLayerUsedByImage = errors.New("layer is in use by an image")
-	// ErrLayerUsedByContainer is returned when the caller attempts to delete a layer that is a container's layer.
-	ErrLayerUsedByContainer = errors.New("layer is in use by a container")
-	// ErrImageUsedByContainer is returned when the caller attempts to delete an image that is a container's image.
-	ErrImageUsedByContainer = errors.New("image is in use by a container")
-	// ErrIncompleteOptions is returned when the caller attempts to initialize a Store without providing required information.
-	ErrIncompleteOptions = errors.New("missing necessary StoreOptions")
-	// ErrSizeUnknown is returned when the caller asks for the size of a big data item, but the Store couldn't determine the answer.
-	ErrSizeUnknown = errors.New("size is not known")
-	// ErrStoreIsReadOnly is returned when the caller makes a call to a read-only store that would require modifying its contents.
-	ErrStoreIsReadOnly = errors.New("called a write method on a read-only store")
 	// DefaultStoreOptions is a reasonable default set of options.
 	DefaultStoreOptions StoreOptions
 	stores              []*store
@@ -183,31 +153,6 @@ type Store interface {
 	// by the Store.
 	GraphDriver() (drivers.Driver, error)
 
-	// LayerStore obtains and returns a handle to the writeable layer store
-	// object used by the Store.  Accessing this store directly will bypass
-	// locking and synchronization, so use it with care.
-	LayerStore() (LayerStore, error)
-
-	// ROLayerStore obtains additional read/only layer store objects used
-	// by the Store.  Accessing these stores directly will bypass locking
-	// and synchronization, so use them with care.
-	ROLayerStores() ([]ROLayerStore, error)
-
-	// ImageStore obtains and returns a handle to the writable image store
-	// object used by the Store.  Accessing this store directly will bypass
-	// locking and synchronization, so use it with care.
-	ImageStore() (ImageStore, error)
-
-	// ROImageStores obtains additional read/only image store objects used
-	// by the Store.  Accessing these stores directly will bypass locking
-	// and synchronization, so use them with care.
-	ROImageStores() ([]ROImageStore, error)
-
-	// ContainerStore obtains and returns a handle to the container store
-	// object used by the Store.  Accessing this store directly will bypass
-	// locking and synchronization, so use it with care.
-	ContainerStore() (ContainerStore, error)
-
 	// CreateLayer creates a new layer in the underlying storage driver,
 	// optionally having the specified ID (one will be assigned if none is
 	// specified), with the specified layer (or no layer) as its parent,
@@ -529,6 +474,7 @@ func GetStore(options StoreOptions) (Store, error) {
 	if err := os.MkdirAll(options.RunRoot, 0700); err != nil && !os.IsExist(err) {
 		return nil, err
 	}
+
 	for _, subdir := range []string{} {
 		if err := os.MkdirAll(filepath.Join(options.RunRoot, subdir), 0700); err != nil && !os.IsExist(err) {
 			return nil, err
@@ -664,6 +610,9 @@ func (s *store) GraphDriver() (drivers.Driver, error) {
 	return s.getGraphDriver()
 }
 
+// LayerStore obtains and returns a handle to the writeable layer store object
+// used by the Store.  Accessing this store directly will bypass locking and
+// synchronization, so it is not a part of the exported Store interface.
 func (s *store) LayerStore() (LayerStore, error) {
 	s.graphLock.Lock()
 	defer s.graphLock.Unlock()
@@ -696,6 +645,9 @@ func (s *store) LayerStore() (LayerStore, error) {
 	return s.layerStore, nil
 }
 
+// ROLayerStores obtains additional read/only layer store objects used by the
+// Store.  Accessing these stores directly will bypass locking and
+// synchronization, so it is not part of the exported Store interface.
 func (s *store) ROLayerStores() ([]ROLayerStore, error) {
 	s.graphLock.Lock()
 	defer s.graphLock.Unlock()
@@ -722,6 +674,9 @@ func (s *store) ROLayerStores() ([]ROLayerStore, error) {
 	return s.roLayerStores, nil
 }
 
+// ImageStore obtains and returns a handle to the writable image store object
+// used by the Store.  Accessing this store directly will bypass locking and
+// synchronization, so it is not a part of the exported Store interface.
 func (s *store) ImageStore() (ImageStore, error) {
 	if s.imageStore != nil {
 		return s.imageStore, nil
@@ -729,6 +684,9 @@ func (s *store) ImageStore() (ImageStore, error) {
 	return nil, ErrLoadError
 }
 
+// ROImageStores obtains additional read/only image store objects used by the
+// Store.  Accessing these stores directly will bypass locking and
+// synchronization, so it is not a part of the exported Store interface.
 func (s *store) ROImageStores() ([]ROImageStore, error) {
 	if len(s.roImageStores) != 0 {
 		return s.roImageStores, nil
@@ -749,6 +707,9 @@ func (s *store) ROImageStores() ([]ROImageStore, error) {
 	return s.roImageStores, nil
 }
 
+// ContainerStore obtains and returns a handle to the container store object
+// used by the Store.  Accessing this store directly will bypass locking and
+// synchronization, so it is not a part of the exported Store interface.
 func (s *store) ContainerStore() (ContainerStore, error) {
 	if s.containerStore != nil {
 		return s.containerStore, nil