Bump github.com/containers/storage@d10d868

Update the vendored copy of github.com/containers/storage to revision
d10d8680af74070b362637408a7fe28c4b1f1eff.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

lock.json

@@ -1,5 +1,5 @@
 {
-    "memo": "e99fe9f7a283d8fb8e0ec8b05fa68d01a7dfa4c7c48b6e85a84986a079685711",
+    "memo": "5791d48b7e77e9f18a26535dfb184838f1d863f5d364fc9907cf16b6013e9846",
     "projects": [
         {
             "name": "cloud.google.com/go",
@@ -117,13 +117,13 @@
         {
             "name": "github.com/containers/storage",
             "branch": "master",
-            "revision": "ff48947baaf205756dd67a00ac688d694a778ef6",
+            "revision": "d10d8680af74070b362637408a7fe28c4b1f1eff",
             "packages": [
                 "drivers",
                 "drivers/aufs",
                 "drivers/btrfs",
                 "drivers/devmapper",
-                "drivers/overlay2",
+                "drivers/overlay",
                 "drivers/register",
                 "drivers/vfs",
                 "drivers/windows",
@@ -149,6 +149,7 @@
                 "pkg/reexec",
                 "pkg/stringid",
                 "pkg/system",
+                "pkg/truncindex",
                 "storage",
                 "storageversion"
             ]

vendor/github.com/containers/storage/cmd/oci-storage/container.go

@@ -20,25 +20,10 @@ func container(flags *mflag.FlagSet, action string, m storage.Store, args []stri
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		return 1
 	}
-	containers, err := m.Containers()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		return 1
-	}
-	matches := []storage.Container{}
-	for _, container := range containers {
-	nextContainer:
-		for _, arg := range args {
-			if container.ID == arg {
-				matches = append(matches, container)
-				break nextContainer
-			}
-			for _, name := range container.Names {
-				if name == arg {
-					matches = append(matches, container)
-					break nextContainer
-				}
-			}
+	matches := []*storage.Container{}
+	for _, arg := range args {
+		if container, err := m.GetContainer(arg); err == nil {
+			matches = append(matches, container)
 		}
 	}
 	if jsonOutput {

vendor/github.com/containers/storage/cmd/oci-storage/image.go

@@ -15,25 +15,10 @@ var (
 )
 
 func image(flags *mflag.FlagSet, action string, m storage.Store, args []string) int {
-	images, err := m.Images()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		return 1
-	}
-	matched := []storage.Image{}
-	for _, image := range images {
-	nextImage:
-		for _, arg := range args {
-			if image.ID == arg {
-				matched = append(matched, image)
-				break nextImage
-			}
-			for _, name := range image.Names {
-				if name == arg {
-					matched = append(matched, image)
-					break nextImage
-				}
-			}
+	matched := []*storage.Image{}
+	for _, arg := range args {
+		if image, err := m.GetImage(arg); err == nil {
+			matched = append(matched, image)
 		}
 	}
 	if jsonOutput {

vendor/github.com/containers/storage/drivers/overlay/mount.go

@@ -1,6 +1,6 @@
 // +build linux
 
-package overlay2
+package overlay
 
 import (
 	"bytes"

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -1,6 +1,6 @@
 // +build linux
 
-package overlay2
+package overlay
 
 import (
 	"bufio"
@@ -61,10 +61,9 @@ var (
 // that mounts do not fail due to length.
 
 const (
-	driverName = "overlay2"
-	linkDir    = "l"
-	lowerFile  = "lower"
-	maxDepth   = 128
+	linkDir   = "l"
+	lowerFile = "lower"
+	maxDepth  = 128
 
 	// idLength represents the number of random characters
 	// which can be used to create the unique link identifer
@@ -78,6 +77,7 @@ const (
 
 // Driver contains information about the home directory and the list of active mounts that are created using this driver.
 type Driver struct {
+	name    string
 	home    string
 	uidMaps []idtools.IDMap
 	gidMaps []idtools.IDMap
@@ -87,13 +87,13 @@ type Driver struct {
 var backingFs = "<unknown>"
 
 func init() {
-	graphdriver.Register(driverName, Init)
+	graphdriver.Register("overlay", InitAsOverlay)
+	graphdriver.Register("overlay2", InitAsOverlay2)
 }
 
-// Init returns the a native diff driver for overlay filesystem.
-// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
-// If a overlay filesystem is not supported over a existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
-func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
+// InitWithName returns the a naive diff driver for the overlay filesystem,
+// which returns the passed-in name when asked which driver it is.
+func InitWithName(name, home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
 	opts, err := parseOptions(options)
 	if err != nil {
 		return nil, err
@@ -112,7 +112,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		if !opts.overrideKernelCheck {
 			return nil, graphdriver.ErrNotSupported
 		}
-		logrus.Warnf("Using pre-4.0.0 kernel for overlay2, mount failures may require kernel update")
+		logrus.Warnf("Using pre-4.0.0 kernel for overlay, mount failures may require kernel update")
 	}
 
 	fsMagic, err := graphdriver.GetFSMagic(home)
@@ -126,7 +126,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 	// check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs
 	switch fsMagic {
 	case graphdriver.FsMagicBtrfs, graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs:
-		logrus.Errorf("'overlay2' is not supported over %s", backingFs)
+		logrus.Errorf("'overlay' is not supported over %s", backingFs)
 		return nil, graphdriver.ErrIncompatibleFS
 	}
 
@@ -144,6 +144,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 	}
 
 	d := &Driver{
+		name:    name,
 		home:    home,
 		uidMaps: uidMaps,
 		gidMaps: gidMaps,
@@ -153,6 +154,20 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 	return d, nil
 }
 
+// InitAsOverlay returns the a naive diff driver for overlay filesystem.
+// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
+// If a overlay filesystem is not supported over a existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
+func InitAsOverlay(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
+	return InitWithName("overlay", home, options, uidMaps, gidMaps)
+}
+
+// InitAsOverlay2 returns the a naive diff driver for overlay filesystem.
+// If overlay filesystem is not supported on the host, graphdriver.ErrNotSupported is returned as error.
+// If a overlay filesystem is not supported over a existing filesystem then error graphdriver.ErrIncompatibleFS is returned.
+func InitAsOverlay2(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
+	return InitWithName("overlay2", home, options, uidMaps, gidMaps)
+}
+
 type overlayOptions struct {
 	overrideKernelCheck bool
 }
@@ -166,13 +181,13 @@ func parseOptions(options []string) (*overlayOptions, error) {
 		}
 		key = strings.ToLower(key)
 		switch key {
-		case "overlay2.override_kernel_check":
+		case "overlay.override_kernel_check", "overlay2.override_kernel_check":
 			o.overrideKernelCheck, err = strconv.ParseBool(val)
 			if err != nil {
 				return nil, err
 			}
 		default:
-			return nil, fmt.Errorf("overlay2: Unknown option %s", key)
+			return nil, fmt.Errorf("overlay: Unknown option %s", key)
 		}
 	}
 	return o, nil
@@ -200,7 +215,7 @@ func supportsOverlay() error {
 }
 
 func (d *Driver) String() string {
-	return driverName
+	return d.name
 }
 
 // Status returns current driver information in a two dimensional string array.

vendor/github.com/containers/storage/drivers/overlay/overlay_test.go

@@ -1,6 +1,6 @@
 // +build linux
 
-package overlay2
+package overlay
 
 import (
 	"os"
@@ -13,6 +13,8 @@ import (
 	"github.com/containers/storage/pkg/reexec"
 )
 
+const driverName = "overlay"
+
 func init() {
 	// Do not sure chroot to speed run time and allow archive
 	// errors or hangs to be debugged directly from the test process.

vendor/github.com/containers/storage/drivers/overlay/overlay_unsupported.go

@@ -1,3 +1,3 @@
 // +build !linux
 
-package overlay2
+package overlay

vendor/github.com/containers/storage/drivers/overlay/randomid.go

@@ -1,6 +1,6 @@
 // +build linux
 
-package overlay2
+package overlay
 
 import (
 	"crypto/rand"

vendor/github.com/containers/storage/drivers/register/register_overlay.go

@@ -3,6 +3,6 @@
 package register
 
 import (
-	// register the overlay2 graphdriver
-	_ "github.com/containers/storage/drivers/overlay2"
+	// register the overlay graphdriver
+	_ "github.com/containers/storage/drivers/overlay"
 )

vendor/github.com/containers/storage/hack/make/cross

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-CROSSPLATFORMS="linux/amd64 linux/386 linux/arm"
+CROSSPLATFORMS="linux/amd64 linux/386 linux/arm darwin/amd64"
 BUILDTAGS+=" exclude_graphdriver_devicemapper"
 
 for platform in $CROSSPLATFORMS; do

vendor/github.com/containers/storage/pkg/truncindex/truncindex.go

@@ -0,0 +1,137 @@
+// Package truncindex provides a general 'index tree', used by Docker
+// in order to be able to reference containers by only a few unambiguous
+// characters of their id.
+package truncindex
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/tchap/go-patricia/patricia"
+)
+
+var (
+	// ErrEmptyPrefix is an error returned if the prefix was empty.
+	ErrEmptyPrefix = errors.New("Prefix can't be empty")
+
+	// ErrIllegalChar is returned when a space is in the ID
+	ErrIllegalChar = errors.New("illegal character: ' '")
+
+	// ErrNotExist is returned when ID or its prefix not found in index.
+	ErrNotExist = errors.New("ID does not exist")
+)
+
+// ErrAmbiguousPrefix is returned if the prefix was ambiguous
+// (multiple ids for the prefix).
+type ErrAmbiguousPrefix struct {
+	prefix string
+}
+
+func (e ErrAmbiguousPrefix) Error() string {
+	return fmt.Sprintf("Multiple IDs found with provided prefix: %s", e.prefix)
+}
+
+// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
+// This is used to retrieve image and container IDs by more convenient shorthand prefixes.
+type TruncIndex struct {
+	sync.RWMutex
+	trie *patricia.Trie
+	ids  map[string]struct{}
+}
+
+// NewTruncIndex creates a new TruncIndex and initializes with a list of IDs.
+func NewTruncIndex(ids []string) (idx *TruncIndex) {
+	idx = &TruncIndex{
+		ids: make(map[string]struct{}),
+
+		// Change patricia max prefix per node length,
+		// because our len(ID) always 64
+		trie: patricia.NewTrie(patricia.MaxPrefixPerNode(64)),
+	}
+	for _, id := range ids {
+		idx.addID(id)
+	}
+	return
+}
+
+func (idx *TruncIndex) addID(id string) error {
+	if strings.Contains(id, " ") {
+		return ErrIllegalChar
+	}
+	if id == "" {
+		return ErrEmptyPrefix
+	}
+	if _, exists := idx.ids[id]; exists {
+		return fmt.Errorf("id already exists: '%s'", id)
+	}
+	idx.ids[id] = struct{}{}
+	if inserted := idx.trie.Insert(patricia.Prefix(id), struct{}{}); !inserted {
+		return fmt.Errorf("failed to insert id: %s", id)
+	}
+	return nil
+}
+
+// Add adds a new ID to the TruncIndex.
+func (idx *TruncIndex) Add(id string) error {
+	idx.Lock()
+	defer idx.Unlock()
+	if err := idx.addID(id); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Delete removes an ID from the TruncIndex. If there are multiple IDs
+// with the given prefix, an error is thrown.
+func (idx *TruncIndex) Delete(id string) error {
+	idx.Lock()
+	defer idx.Unlock()
+	if _, exists := idx.ids[id]; !exists || id == "" {
+		return fmt.Errorf("no such id: '%s'", id)
+	}
+	delete(idx.ids, id)
+	if deleted := idx.trie.Delete(patricia.Prefix(id)); !deleted {
+		return fmt.Errorf("no such id: '%s'", id)
+	}
+	return nil
+}
+
+// Get retrieves an ID from the TruncIndex. If there are multiple IDs
+// with the given prefix, an error is thrown.
+func (idx *TruncIndex) Get(s string) (string, error) {
+	if s == "" {
+		return "", ErrEmptyPrefix
+	}
+	var (
+		id string
+	)
+	subTreeVisitFunc := func(prefix patricia.Prefix, item patricia.Item) error {
+		if id != "" {
+			// we haven't found the ID if there are two or more IDs
+			id = ""
+			return ErrAmbiguousPrefix{prefix: string(prefix)}
+		}
+		id = string(prefix)
+		return nil
+	}
+
+	idx.RLock()
+	defer idx.RUnlock()
+	if err := idx.trie.VisitSubtree(patricia.Prefix(s), subTreeVisitFunc); err != nil {
+		return "", err
+	}
+	if id != "" {
+		return id, nil
+	}
+	return "", ErrNotExist
+}
+
+// Iterate iterates over all stored IDs, and passes each of them to the given handler.
+func (idx *TruncIndex) Iterate(handler func(id string)) {
+	idx.trie.Visit(func(prefix patricia.Prefix, item patricia.Item) error {
+		handler(string(prefix))
+		return nil
+	})
+}

vendor/github.com/containers/storage/pkg/truncindex/truncindex_test.go

@@ -0,0 +1,429 @@
+package truncindex
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/containers/storage/pkg/stringid"
+)
+
+// Test the behavior of TruncIndex, an index for querying IDs from a non-conflicting prefix.
+func TestTruncIndex(t *testing.T) {
+	ids := []string{}
+	index := NewTruncIndex(ids)
+	// Get on an empty index
+	if _, err := index.Get("foobar"); err == nil {
+		t.Fatal("Get on an empty index should return an error")
+	}
+
+	// Spaces should be illegal in an id
+	if err := index.Add("I have a space"); err == nil {
+		t.Fatalf("Adding an id with ' ' should return an error")
+	}
+
+	id := "99b36c2c326ccc11e726eee6ee78a0baf166ef96"
+	// Add an id
+	if err := index.Add(id); err != nil {
+		t.Fatal(err)
+	}
+
+	// Add an empty id (should fail)
+	if err := index.Add(""); err == nil {
+		t.Fatalf("Adding an empty id should return an error")
+	}
+
+	// Get a non-existing id
+	assertIndexGet(t, index, "abracadabra", "", true)
+	// Get an empty id
+	assertIndexGet(t, index, "", "", true)
+	// Get the exact id
+	assertIndexGet(t, index, id, id, false)
+	// The first letter should match
+	assertIndexGet(t, index, id[:1], id, false)
+	// The first half should match
+	assertIndexGet(t, index, id[:len(id)/2], id, false)
+	// The second half should NOT match
+	assertIndexGet(t, index, id[len(id)/2:], "", true)
+
+	id2 := id[:6] + "blabla"
+	// Add an id
+	if err := index.Add(id2); err != nil {
+		t.Fatal(err)
+	}
+	// Both exact IDs should work
+	assertIndexGet(t, index, id, id, false)
+	assertIndexGet(t, index, id2, id2, false)
+
+	// 6 characters or less should conflict
+	assertIndexGet(t, index, id[:6], "", true)
+	assertIndexGet(t, index, id[:4], "", true)
+	assertIndexGet(t, index, id[:1], "", true)
+
+	// An ambiguous id prefix should return an error
+	if _, err := index.Get(id[:4]); err == nil {
+		t.Fatal("An ambiguous id prefix should return an error")
+	}
+
+	// 7 characters should NOT conflict
+	assertIndexGet(t, index, id[:7], id, false)
+	assertIndexGet(t, index, id2[:7], id2, false)
+
+	// Deleting a non-existing id should return an error
+	if err := index.Delete("non-existing"); err == nil {
+		t.Fatalf("Deleting a non-existing id should return an error")
+	}
+
+	// Deleting an empty id should return an error
+	if err := index.Delete(""); err == nil {
+		t.Fatal("Deleting an empty id should return an error")
+	}
+
+	// Deleting id2 should remove conflicts
+	if err := index.Delete(id2); err != nil {
+		t.Fatal(err)
+	}
+	// id2 should no longer work
+	assertIndexGet(t, index, id2, "", true)
+	assertIndexGet(t, index, id2[:7], "", true)
+	assertIndexGet(t, index, id2[:11], "", true)
+
+	// conflicts between id and id2 should be gone
+	assertIndexGet(t, index, id[:6], id, false)
+	assertIndexGet(t, index, id[:4], id, false)
+	assertIndexGet(t, index, id[:1], id, false)
+
+	// non-conflicting substrings should still not conflict
+	assertIndexGet(t, index, id[:7], id, false)
+	assertIndexGet(t, index, id[:15], id, false)
+	assertIndexGet(t, index, id, id, false)
+
+	assertIndexIterate(t)
+}
+
+func assertIndexIterate(t *testing.T) {
+	ids := []string{
+		"19b36c2c326ccc11e726eee6ee78a0baf166ef96",
+		"28b36c2c326ccc11e726eee6ee78a0baf166ef96",
+		"37b36c2c326ccc11e726eee6ee78a0baf166ef96",
+		"46b36c2c326ccc11e726eee6ee78a0baf166ef96",
+	}
+
+	index := NewTruncIndex(ids)
+
+	index.Iterate(func(targetId string) {
+		for _, id := range ids {
+			if targetId == id {
+				return
+			}
+		}
+
+		t.Fatalf("An unknown ID '%s'", targetId)
+	})
+}
+
+func assertIndexGet(t *testing.T, index *TruncIndex, input, expectedResult string, expectError bool) {
+	if result, err := index.Get(input); err != nil && !expectError {
+		t.Fatalf("Unexpected error getting '%s': %s", input, err)
+	} else if err == nil && expectError {
+		t.Fatalf("Getting '%s' should return an error, not '%s'", input, result)
+	} else if result != expectedResult {
+		t.Fatalf("Getting '%s' returned '%s' instead of '%s'", input, result, expectedResult)
+	}
+}
+
+func BenchmarkTruncIndexAdd100(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 100; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexAdd250(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 250; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexAdd500(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 500; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexGet100(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 100; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	index := NewTruncIndex([]string{})
+	for _, id := range testSet {
+		if err := index.Add(id); err != nil {
+			b.Fatal(err)
+		}
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexGet250(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 250; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	index := NewTruncIndex([]string{})
+	for _, id := range testSet {
+		if err := index.Add(id); err != nil {
+			b.Fatal(err)
+		}
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexGet500(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 500; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	index := NewTruncIndex([]string{})
+	for _, id := range testSet {
+		if err := index.Add(id); err != nil {
+			b.Fatal(err)
+		}
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexDelete100(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 100; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+		b.StartTimer()
+		for _, id := range testSet {
+			if err := index.Delete(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexDelete250(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 250; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+		b.StartTimer()
+		for _, id := range testSet {
+			if err := index.Delete(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexDelete500(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 500; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+		b.StartTimer()
+		for _, id := range testSet {
+			if err := index.Delete(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexNew100(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 100; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		NewTruncIndex(testSet)
+	}
+}
+
+func BenchmarkTruncIndexNew250(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 250; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		NewTruncIndex(testSet)
+	}
+}
+
+func BenchmarkTruncIndexNew500(b *testing.B) {
+	var testSet []string
+	for i := 0; i < 500; i++ {
+		testSet = append(testSet, stringid.GenerateNonCryptoID())
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		NewTruncIndex(testSet)
+	}
+}
+
+func BenchmarkTruncIndexAddGet100(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 500; i++ {
+		id := stringid.GenerateNonCryptoID()
+		testSet = append(testSet, id)
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexAddGet250(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 500; i++ {
+		id := stringid.GenerateNonCryptoID()
+		testSet = append(testSet, id)
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}
+
+func BenchmarkTruncIndexAddGet500(b *testing.B) {
+	var testSet []string
+	var testKeys []string
+	for i := 0; i < 500; i++ {
+		id := stringid.GenerateNonCryptoID()
+		testSet = append(testSet, id)
+		l := rand.Intn(12) + 12
+		testKeys = append(testKeys, id[:l])
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		index := NewTruncIndex([]string{})
+		for _, id := range testSet {
+			if err := index.Add(id); err != nil {
+				b.Fatal(err)
+			}
+		}
+		for _, id := range testKeys {
+			if res, err := index.Get(id); err != nil {
+				b.Fatal(res, err)
+			}
+		}
+	}
+}

vendor/github.com/containers/storage/storage/containers.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/stringid"
+	"github.com/containers/storage/pkg/truncindex"
 )
 
 var (
@@ -93,6 +94,7 @@ type containerStore struct {
 	lockfile   Locker
 	dir        string
 	containers []Container
+	idindex    *truncindex.TruncIndex
 	byid       map[string]*Container
 	bylayer    map[string]*Container
 	byname     map[string]*Container
@@ -123,10 +125,12 @@ func (r *containerStore) Load() error {
 	}
 	containers := []Container{}
 	layers := make(map[string]*Container)
+	idlist := []string{}
 	ids := make(map[string]*Container)
 	names := make(map[string]*Container)
 	if err = json.Unmarshal(data, &containers); len(data) == 0 || err == nil {
 		for n, container := range containers {
+			idlist = append(idlist, container.ID)
 			ids[container.ID] = &containers[n]
 			layers[container.LayerID] = &containers[n]
 			for _, name := range container.Names {
@@ -139,6 +143,7 @@ func (r *containerStore) Load() error {
 		}
 	}
 	r.containers = containers
+	r.idindex = truncindex.NewTruncIndex(idlist)
 	r.byid = ids
 	r.bylayer = layers
 	r.byname = names
@@ -185,30 +190,35 @@ func newContainerStore(dir string) (ContainerStore, error) {
 	return &cstore, nil
 }
 
-func (r *containerStore) ClearFlag(id string, flag string) error {
-	if container, ok := r.byname[id]; ok {
-		id = container.ID
+func (r *containerStore) lookup(id string) (*Container, bool) {
+	if container, ok := r.byid[id]; ok {
+		return container, ok
+	} else if container, ok := r.byname[id]; ok {
+		return container, ok
 	} else if container, ok := r.bylayer[id]; ok {
-		id = container.ID
+		return container, ok
+	} else if longid, err := r.idindex.Get(id); err == nil {
+		if container, ok := r.byid[longid]; ok {
+			return container, ok
+		}
 	}
-	if _, ok := r.byid[id]; !ok {
+	return nil, false
+}
+
+func (r *containerStore) ClearFlag(id string, flag string) error {
+	container, ok := r.lookup(id)
+	if !ok {
 		return ErrContainerUnknown
 	}
-	container := r.byid[id]
 	delete(container.Flags, flag)
 	return r.Save()
 }
 
 func (r *containerStore) SetFlag(id string, flag string, value interface{}) error {
-	if container, ok := r.byname[id]; ok {
-		id = container.ID
-	} else if container, ok := r.bylayer[id]; ok {
-		id = container.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	container, ok := r.lookup(id)
+	if !ok {
 		return ErrContainerUnknown
 	}
-	container := r.byid[id]
 	container.Flags[flag] = value
 	return r.Save()
 }
@@ -244,6 +254,7 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
 		r.containers = append(r.containers, newContainer)
 		container = &r.containers[len(r.containers)-1]
 		r.byid[id] = container
+		r.idindex.Add(id)
 		r.bylayer[layer] = container
 		for _, name := range names {
 			r.byname[name] = container
@@ -254,24 +265,14 @@ func (r *containerStore) Create(id string, names []string, image, layer, metadat
 }
 
 func (r *containerStore) GetMetadata(id string) (string, error) {
-	if container, ok := r.byname[id]; ok {
-		id = container.ID
-	} else if container, ok := r.bylayer[id]; ok {
-		id = container.ID
-	}
-	if container, ok := r.byid[id]; ok {
+	if container, ok := r.lookup(id); ok {
 		return container.Metadata, nil
 	}
 	return "", ErrContainerUnknown
 }
 
 func (r *containerStore) SetMetadata(id, metadata string) error {
-	if container, ok := r.byname[id]; ok {
-		id = container.ID
-	} else if container, ok := r.bylayer[id]; ok {
-		id = container.ID
-	}
-	if container, ok := r.byid[id]; ok {
+	if container, ok := r.lookup(id); ok {
 		container.Metadata = metadata
 		return r.Save()
 	}
@@ -279,22 +280,11 @@ func (r *containerStore) SetMetadata(id, metadata string) error {
 }
 
 func (r *containerStore) removeName(container *Container, name string) {
-	newNames := []string{}
-	for _, oldName := range container.Names {
-		if oldName != name {
-			newNames = append(newNames, oldName)
-		}
-	}
-	container.Names = newNames
+	container.Names = stringSliceWithoutValue(container.Names, name)
 }
 
 func (r *containerStore) SetNames(id string, names []string) error {
-	if container, ok := r.byname[id]; ok {
-		id = container.ID
-	} else if container, ok := r.bylayer[id]; ok {
-		id = container.ID
-	}
-	if container, ok := r.byid[id]; ok {
+	if container, ok := r.lookup(id); ok {
 		for _, name := range container.Names {
 			delete(r.byname, name)
 		}
@@ -311,133 +301,104 @@ func (r *containerStore) SetNames(id string, names []string) error {
 }
 
 func (r *containerStore) Delete(id string) error {
-	if container, ok := r.byname[id]; ok {
-		id = container.ID
-	} else if container, ok := r.bylayer[id]; ok {
-		id = container.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	container, ok := r.lookup(id)
+	if !ok {
 		return ErrContainerUnknown
 	}
-	if container, ok := r.byid[id]; ok {
-		newContainers := []Container{}
-		for _, candidate := range r.containers {
-			if candidate.ID != id {
-				newContainers = append(newContainers, candidate)
-			}
-		}
-		delete(r.byid, container.ID)
-		delete(r.bylayer, container.LayerID)
-		for _, name := range container.Names {
-			delete(r.byname, name)
-		}
-		r.containers = newContainers
-		if err := r.Save(); err != nil {
-			return err
-		}
-		if err := os.RemoveAll(r.datadir(id)); err != nil {
-			return err
+	id = container.ID
+	newContainers := []Container{}
+	for _, candidate := range r.containers {
+		if candidate.ID != id {
+			newContainers = append(newContainers, candidate)
 		}
 	}
+	delete(r.byid, id)
+	r.idindex.Delete(id)
+	delete(r.bylayer, container.LayerID)
+	for _, name := range container.Names {
+		delete(r.byname, name)
+	}
+	r.containers = newContainers
+	if err := r.Save(); err != nil {
+		return err
+	}
+	if err := os.RemoveAll(r.datadir(id)); err != nil {
+		return err
+	}
 	return nil
 }
 
 func (r *containerStore) Get(id string) (*Container, error) {
-	if c, ok := r.byname[id]; ok {
-		return c, nil
-	} else if c, ok := r.bylayer[id]; ok {
-		return c, nil
-	}
-	if c, ok := r.byid[id]; ok {
-		return c, nil
+	if container, ok := r.lookup(id); ok {
+		return container, nil
 	}
 	return nil, ErrContainerUnknown
 }
 
 func (r *containerStore) Lookup(name string) (id string, err error) {
-	container, ok := r.byname[name]
-	if !ok {
-		container, ok = r.byid[name]
-		if !ok {
-			return "", ErrContainerUnknown
-		}
+	if container, ok := r.lookup(name); ok {
+		return container.ID, nil
 	}
-	return container.ID, nil
+	return "", ErrContainerUnknown
 }
 
 func (r *containerStore) Exists(id string) bool {
-	if _, ok := r.byname[id]; ok {
-		return true
-	}
-	if _, ok := r.bylayer[id]; ok {
-		return true
-	}
-	if _, ok := r.byid[id]; ok {
-		return true
-	}
-	return false
+	_, ok := r.lookup(id)
+	return ok
 }
 
 func (r *containerStore) GetBigData(id, key string) ([]byte, error) {
-	if img, ok := r.byname[id]; ok {
-		id = img.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	c, ok := r.lookup(id)
+	if !ok {
 		return nil, ErrContainerUnknown
 	}
-	return ioutil.ReadFile(r.datapath(id, key))
+	return ioutil.ReadFile(r.datapath(c.ID, key))
 }
 
 func (r *containerStore) GetBigDataSize(id, key string) (int64, error) {
-	if img, ok := r.byname[id]; ok {
-		id = img.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	c, ok := r.lookup(id)
+	if !ok {
 		return -1, ErrContainerUnknown
 	}
-	if size, ok := r.byid[id].BigDataSizes[key]; ok {
+	if size, ok := c.BigDataSizes[key]; ok {
 		return size, nil
 	}
 	return -1, ErrSizeUnknown
 }
 
 func (r *containerStore) GetBigDataNames(id string) ([]string, error) {
-	if img, ok := r.byname[id]; ok {
-		id = img.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	c, ok := r.lookup(id)
+	if !ok {
 		return nil, ErrContainerUnknown
 	}
-	return r.byid[id].BigDataNames, nil
+	return c.BigDataNames, nil
 }
 
 func (r *containerStore) SetBigData(id, key string, data []byte) error {
-	if img, ok := r.byname[id]; ok {
-		id = img.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	c, ok := r.lookup(id)
+	if !ok {
 		return ErrContainerUnknown
 	}
-	if err := os.MkdirAll(r.datadir(id), 0700); err != nil {
+	if err := os.MkdirAll(r.datadir(c.ID), 0700); err != nil {
 		return err
 	}
-	err := ioutils.AtomicWriteFile(r.datapath(id, key), data, 0600)
+	err := ioutils.AtomicWriteFile(r.datapath(c.ID, key), data, 0600)
 	if err == nil {
 		save := false
-		oldSize, ok := r.byid[id].BigDataSizes[key]
-		r.byid[id].BigDataSizes[key] = int64(len(data))
-		if !ok || oldSize != r.byid[id].BigDataSizes[key] {
+		oldSize, ok := c.BigDataSizes[key]
+		c.BigDataSizes[key] = int64(len(data))
+		if !ok || oldSize != c.BigDataSizes[key] {
 			save = true
 		}
 		add := true
-		for _, name := range r.byid[id].BigDataNames {
+		for _, name := range c.BigDataNames {
 			if name == key {
 				add = false
 				break
 			}
 		}
 		if add {
-			r.byid[id].BigDataNames = append(r.byid[id].BigDataNames, key)
+			c.BigDataNames = append(c.BigDataNames, key)
 			save = true
 		}
 		if save {

vendor/github.com/containers/storage/storage/images.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/stringid"
+	"github.com/containers/storage/pkg/truncindex"
 )
 
 var (
@@ -88,6 +89,7 @@ type imageStore struct {
 	lockfile Locker
 	dir      string
 	images   []Image
+	idindex  *truncindex.TruncIndex
 	byid     map[string]*Image
 	byname   map[string]*Image
 }
@@ -116,11 +118,13 @@ func (r *imageStore) Load() error {
 		return err
 	}
 	images := []Image{}
+	idlist := []string{}
 	ids := make(map[string]*Image)
 	names := make(map[string]*Image)
 	if err = json.Unmarshal(data, &images); len(data) == 0 || err == nil {
 		for n, image := range images {
 			ids[image.ID] = &images[n]
+			idlist = append(idlist, image.ID)
 			for _, name := range image.Names {
 				if conflict, ok := names[name]; ok {
 					r.removeName(conflict, name)
@@ -131,6 +135,7 @@ func (r *imageStore) Load() error {
 		}
 	}
 	r.images = images
+	r.idindex = truncindex.NewTruncIndex(idlist)
 	r.byid = ids
 	r.byname = names
 	if needSave {
@@ -175,26 +180,32 @@ func newImageStore(dir string) (ImageStore, error) {
 	return &istore, nil
 }
 
-func (r *imageStore) ClearFlag(id string, flag string) error {
-	if image, ok := r.byname[id]; ok {
-		id = image.ID
+func (r *imageStore) lookup(id string) (*Image, bool) {
+	if image, ok := r.byid[id]; ok {
+		return image, ok
+	} else if image, ok := r.byname[id]; ok {
+		return image, ok
+	} else if longid, err := r.idindex.Get(id); err == nil {
+		image, ok := r.byid[longid]
+		return image, ok
 	}
-	if _, ok := r.byid[id]; !ok {
+	return nil, false
+}
+
+func (r *imageStore) ClearFlag(id string, flag string) error {
+	image, ok := r.lookup(id)
+	if !ok {
 		return ErrImageUnknown
 	}
-	image := r.byid[id]
 	delete(image.Flags, flag)
 	return r.Save()
 }
 
 func (r *imageStore) SetFlag(id string, flag string, value interface{}) error {
-	if image, ok := r.byname[id]; ok {
-		id = image.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	image, ok := r.lookup(id)
+	if !ok {
 		return ErrImageUnknown
 	}
-	image := r.byid[id]
 	image.Flags[flag] = value
 	return r.Save()
 }
@@ -228,6 +239,7 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string) (
 		}
 		r.images = append(r.images, newImage)
 		image = &r.images[len(r.images)-1]
+		r.idindex.Add(id)
 		r.byid[id] = image
 		for _, name := range names {
 			r.byname[name] = image
@@ -238,20 +250,14 @@ func (r *imageStore) Create(id string, names []string, layer, metadata string) (
 }
 
 func (r *imageStore) GetMetadata(id string) (string, error) {
-	if image, ok := r.byname[id]; ok {
-		id = image.ID
-	}
-	if image, ok := r.byid[id]; ok {
+	if image, ok := r.lookup(id); ok {
 		return image.Metadata, nil
 	}
 	return "", ErrImageUnknown
 }
 
 func (r *imageStore) SetMetadata(id, metadata string) error {
-	if image, ok := r.byname[id]; ok {
-		id = image.ID
-	}
-	if image, ok := r.byid[id]; ok {
+	if image, ok := r.lookup(id); ok {
 		image.Metadata = metadata
 		return r.Save()
 	}
@@ -259,20 +265,11 @@ func (r *imageStore) SetMetadata(id, metadata string) error {
 }
 
 func (r *imageStore) removeName(image *Image, name string) {
-	newNames := []string{}
-	for _, oldName := range image.Names {
-		if oldName != name {
-			newNames = append(newNames, oldName)
-		}
-	}
-	image.Names = newNames
+	image.Names = stringSliceWithoutValue(image.Names, name)
 }
 
 func (r *imageStore) SetNames(id string, names []string) error {
-	if image, ok := r.byname[id]; ok {
-		id = image.ID
-	}
-	if image, ok := r.byid[id]; ok {
+	if image, ok := r.lookup(id); ok {
 		for _, name := range image.Names {
 			delete(r.byname, name)
 		}
@@ -289,125 +286,103 @@ func (r *imageStore) SetNames(id string, names []string) error {
 }
 
 func (r *imageStore) Delete(id string) error {
-	if image, ok := r.byname[id]; ok {
-		id = image.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	image, ok := r.lookup(id)
+	if !ok {
 		return ErrImageUnknown
 	}
-	if image, ok := r.byid[id]; ok {
-		newImages := []Image{}
-		for _, candidate := range r.images {
-			if candidate.ID != id {
-				newImages = append(newImages, candidate)
-			}
-		}
-		delete(r.byid, image.ID)
-		for _, name := range image.Names {
-			delete(r.byname, name)
-		}
-		r.images = newImages
-		if err := r.Save(); err != nil {
-			return err
-		}
-		if err := os.RemoveAll(r.datadir(id)); err != nil {
-			return err
+	id = image.ID
+	newImages := []Image{}
+	for _, candidate := range r.images {
+		if candidate.ID != id {
+			newImages = append(newImages, candidate)
 		}
 	}
+	delete(r.byid, id)
+	r.idindex.Delete(id)
+	for _, name := range image.Names {
+		delete(r.byname, name)
+	}
+	r.images = newImages
+	if err := r.Save(); err != nil {
+		return err
+	}
+	if err := os.RemoveAll(r.datadir(id)); err != nil {
+		return err
+	}
 	return nil
 }
 
 func (r *imageStore) Get(id string) (*Image, error) {
-	if image, ok := r.byname[id]; ok {
-		return image, nil
-	}
-	if image, ok := r.byid[id]; ok {
+	if image, ok := r.lookup(id); ok {
 		return image, nil
 	}
 	return nil, ErrImageUnknown
 }
 
 func (r *imageStore) Lookup(name string) (id string, err error) {
-	image, ok := r.byname[name]
-	if !ok {
-		image, ok = r.byid[name]
-		if !ok {
-			return "", ErrImageUnknown
-		}
+	if image, ok := r.lookup(name); ok {
+		return image.ID, nil
 	}
-	return image.ID, nil
+	return "", ErrImageUnknown
 }
 
 func (r *imageStore) Exists(id string) bool {
-	if _, ok := r.byname[id]; ok {
-		return true
-	}
-	if _, ok := r.byid[id]; ok {
-		return true
-	}
-	return false
+	_, ok := r.lookup(id)
+	return ok
 }
 
 func (r *imageStore) GetBigData(id, key string) ([]byte, error) {
-	if img, ok := r.byname[id]; ok {
-		id = img.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	image, ok := r.lookup(id)
+	if !ok {
 		return nil, ErrImageUnknown
 	}
-	return ioutil.ReadFile(r.datapath(id, key))
+	return ioutil.ReadFile(r.datapath(image.ID, key))
 }
 
 func (r *imageStore) GetBigDataSize(id, key string) (int64, error) {
-	if img, ok := r.byname[id]; ok {
-		id = img.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	image, ok := r.lookup(id)
+	if !ok {
 		return -1, ErrImageUnknown
 	}
-	if size, ok := r.byid[id].BigDataSizes[key]; ok {
+	if size, ok := image.BigDataSizes[key]; ok {
 		return size, nil
 	}
 	return -1, ErrSizeUnknown
 }
 
 func (r *imageStore) GetBigDataNames(id string) ([]string, error) {
-	if img, ok := r.byname[id]; ok {
-		id = img.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	image, ok := r.lookup(id)
+	if !ok {
 		return nil, ErrImageUnknown
 	}
-	return r.byid[id].BigDataNames, nil
+	return image.BigDataNames, nil
 }
 
 func (r *imageStore) SetBigData(id, key string, data []byte) error {
-	if img, ok := r.byname[id]; ok {
-		id = img.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	image, ok := r.lookup(id)
+	if !ok {
 		return ErrImageUnknown
 	}
-	if err := os.MkdirAll(r.datadir(id), 0700); err != nil {
+	if err := os.MkdirAll(r.datadir(image.ID), 0700); err != nil {
 		return err
 	}
-	err := ioutils.AtomicWriteFile(r.datapath(id, key), data, 0600)
+	err := ioutils.AtomicWriteFile(r.datapath(image.ID, key), data, 0600)
 	if err == nil {
 		add := true
 		save := false
-		oldSize, ok := r.byid[id].BigDataSizes[key]
-		r.byid[id].BigDataSizes[key] = int64(len(data))
-		if !ok || oldSize != r.byid[id].BigDataSizes[key] {
+		oldSize, ok := image.BigDataSizes[key]
+		image.BigDataSizes[key] = int64(len(data))
+		if !ok || oldSize != image.BigDataSizes[key] {
 			save = true
 		}
-		for _, name := range r.byid[id].BigDataNames {
+		for _, name := range image.BigDataNames {
 			if name == key {
 				add = false
 				break
 			}
 		}
 		if add {
-			r.byid[id].BigDataNames = append(r.byid[id].BigDataNames, key)
+			image.BigDataNames = append(image.BigDataNames, key)
 			save = true
 		}
 		if save {

vendor/github.com/containers/storage/storage/layers.go

@@ -15,6 +15,7 @@ import (
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/stringid"
+	"github.com/containers/storage/pkg/truncindex"
 	"github.com/vbatts/tar-split/tar/asm"
 	"github.com/vbatts/tar-split/tar/storage"
 )
@@ -159,6 +160,7 @@ type layerStore struct {
 	driver   drivers.Driver
 	layerdir string
 	layers   []Layer
+	idindex  *truncindex.TruncIndex
 	byid     map[string]*Layer
 	byname   map[string]*Layer
 	byparent map[string][]*Layer
@@ -185,6 +187,7 @@ func (r *layerStore) Load() error {
 		return err
 	}
 	layers := []Layer{}
+	idlist := []string{}
 	ids := make(map[string]*Layer)
 	names := make(map[string]*Layer)
 	mounts := make(map[string]*Layer)
@@ -192,6 +195,7 @@ func (r *layerStore) Load() error {
 	if err = json.Unmarshal(data, &layers); len(data) == 0 || err == nil {
 		for n, layer := range layers {
 			ids[layer.ID] = &layers[n]
+			idlist = append(idlist, layer.ID)
 			for _, name := range layer.Names {
 				if conflict, ok := names[name]; ok {
 					r.removeName(conflict, name)
@@ -224,6 +228,7 @@ func (r *layerStore) Load() error {
 		}
 	}
 	r.layers = layers
+	r.idindex = truncindex.NewTruncIndex(idlist)
 	r.byid = ids
 	r.byname = names
 	r.byparent = parents
@@ -312,26 +317,32 @@ func newLayerStore(rundir string, layerdir string, driver drivers.Driver) (Layer
 	return &rlstore, nil
 }
 
-func (r *layerStore) ClearFlag(id string, flag string) error {
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
+func (r *layerStore) lookup(id string) (*Layer, bool) {
+	if layer, ok := r.byid[id]; ok {
+		return layer, ok
+	} else if layer, ok := r.byname[id]; ok {
+		return layer, ok
+	} else if longid, err := r.idindex.Get(id); err == nil {
+		layer, ok := r.byid[longid]
+		return layer, ok
 	}
-	if _, ok := r.byid[id]; !ok {
+	return nil, false
+}
+
+func (r *layerStore) ClearFlag(id string, flag string) error {
+	layer, ok := r.lookup(id)
+	if !ok {
 		return ErrLayerUnknown
 	}
-	layer := r.byid[id]
 	delete(layer.Flags, flag)
 	return r.Save()
 }
 
 func (r *layerStore) SetFlag(id string, flag string, value interface{}) error {
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	layer, ok := r.lookup(id)
+	if !ok {
 		return ErrLayerUnknown
 	}
-	layer := r.byid[id]
 	layer.Flags[flag] = value
 	return r.Save()
 }
@@ -348,8 +359,10 @@ func (r *layerStore) Put(id, parent string, names []string, mountLabel string, o
 	if err := os.MkdirAll(r.layerdir, 0700); err != nil {
 		return nil, -1, err
 	}
-	if parentLayer, ok := r.byname[parent]; ok {
-		parent = parentLayer.ID
+	if parent != "" {
+		if parentLayer, ok := r.lookup(parent); ok {
+			parent = parentLayer.ID
+		}
 	}
 	if id == "" {
 		id = stringid.GenerateRandomID()
@@ -382,6 +395,7 @@ func (r *layerStore) Put(id, parent string, names []string, mountLabel string, o
 		}
 		r.layers = append(r.layers, newLayer)
 		layer = &r.layers[len(r.layers)-1]
+		r.idindex.Add(id)
 		r.byid[id] = layer
 		for _, name := range names {
 			r.byname[name] = layer
@@ -436,48 +450,39 @@ func (r *layerStore) Create(id, parent string, names []string, mountLabel string
 }
 
 func (r *layerStore) Mount(id, mountLabel string) (string, error) {
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	layer, ok := r.lookup(id)
+	if !ok {
 		return "", ErrLayerUnknown
 	}
-	layer := r.byid[id]
 	if layer.MountCount > 0 {
 		layer.MountCount++
 		return layer.MountPoint, r.Save()
 	}
 	if mountLabel == "" {
-		if layer, ok := r.byid[id]; ok {
-			mountLabel = layer.MountLabel
-		}
+		mountLabel = layer.MountLabel
 	}
 	mountpoint, err := r.driver.Get(id, mountLabel)
 	if mountpoint != "" && err == nil {
-		if layer, ok := r.byid[id]; ok {
-			if layer.MountPoint != "" {
-				delete(r.bymount, layer.MountPoint)
-			}
-			layer.MountPoint = filepath.Clean(mountpoint)
-			layer.MountCount++
-			r.bymount[layer.MountPoint] = layer
-			err = r.Save()
+		if layer.MountPoint != "" {
+			delete(r.bymount, layer.MountPoint)
 		}
+		layer.MountPoint = filepath.Clean(mountpoint)
+		layer.MountCount++
+		r.bymount[layer.MountPoint] = layer
+		err = r.Save()
 	}
 	return mountpoint, err
 }
 
 func (r *layerStore) Unmount(id string) error {
-	if layer, ok := r.bymount[filepath.Clean(id)]; ok {
-		id = layer.ID
+	layer, ok := r.lookup(id)
+	if !ok {
+		layerByMount, ok := r.bymount[filepath.Clean(id)]
+		if !ok {
+			return ErrLayerUnknown
+		}
+		layer = layerByMount
 	}
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
-	}
-	if _, ok := r.byid[id]; !ok {
-		return ErrLayerUnknown
-	}
-	layer := r.byid[id]
 	if layer.MountCount > 1 {
 		layer.MountCount--
 		return r.Save()
@@ -495,20 +500,11 @@ func (r *layerStore) Unmount(id string) error {
 }
 
 func (r *layerStore) removeName(layer *Layer, name string) {
-	newNames := []string{}
-	for _, oldName := range layer.Names {
-		if oldName != name {
-			newNames = append(newNames, oldName)
-		}
-	}
-	layer.Names = newNames
+	layer.Names = stringSliceWithoutValue(layer.Names, name)
 }
 
 func (r *layerStore) SetNames(id string, names []string) error {
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
-	}
-	if layer, ok := r.byid[id]; ok {
+	if layer, ok := r.lookup(id); ok {
 		for _, name := range layer.Names {
 			delete(r.byname, name)
 		}
@@ -525,20 +521,14 @@ func (r *layerStore) SetNames(id string, names []string) error {
 }
 
 func (r *layerStore) GetMetadata(id string) (string, error) {
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
-	}
-	if layer, ok := r.byid[id]; ok {
+	if layer, ok := r.lookup(id); ok {
 		return layer.Metadata, nil
 	}
 	return "", ErrLayerUnknown
 }
 
 func (r *layerStore) SetMetadata(id, metadata string) error {
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
-	}
-	if layer, ok := r.byid[id]; ok {
+	if layer, ok := r.lookup(id); ok {
 		layer.Metadata = metadata
 		return r.Save()
 	}
@@ -550,13 +540,12 @@ func (r *layerStore) tspath(id string) string {
 }
 
 func (r *layerStore) Delete(id string) error {
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
-	}
-	if _, ok := r.byid[id]; !ok {
+	layer, ok := r.lookup(id)
+	if !ok {
 		return ErrLayerUnknown
 	}
-	for r.byid[id].MountCount > 0 {
+	id = layer.ID
+	for layer.MountCount > 0 {
 		if err := r.Unmount(id); err != nil {
 			return err
 		}
@@ -564,66 +553,55 @@ func (r *layerStore) Delete(id string) error {
 	err := r.driver.Remove(id)
 	if err == nil {
 		os.Remove(r.tspath(id))
-		if layer, ok := r.byid[id]; ok {
-			pslice := r.byparent[layer.Parent]
-			newPslice := []*Layer{}
-			for _, candidate := range pslice {
-				if candidate.ID != id {
-					newPslice = append(newPslice, candidate)
-				}
+		pslice := r.byparent[layer.Parent]
+		newPslice := []*Layer{}
+		for _, candidate := range pslice {
+			if candidate.ID != id {
+				newPslice = append(newPslice, candidate)
 			}
-			delete(r.byid, layer.ID)
-			if len(newPslice) > 0 {
-				r.byparent[layer.Parent] = newPslice
-			} else {
-				delete(r.byparent, layer.Parent)
-			}
-			for _, name := range layer.Names {
-				delete(r.byname, name)
-			}
-			if layer.MountPoint != "" {
-				delete(r.bymount, layer.MountPoint)
-			}
-			newLayers := []Layer{}
-			for _, candidate := range r.layers {
-				if candidate.ID != id {
-					newLayers = append(newLayers, candidate)
-				}
-			}
-			r.layers = newLayers
-			if err = r.Save(); err != nil {
-				return err
+		}
+		delete(r.byid, id)
+		r.idindex.Delete(id)
+		if len(newPslice) > 0 {
+			r.byparent[layer.Parent] = newPslice
+		} else {
+			delete(r.byparent, layer.Parent)
+		}
+		for _, name := range layer.Names {
+			delete(r.byname, name)
+		}
+		if layer.MountPoint != "" {
+			delete(r.bymount, layer.MountPoint)
+		}
+		newLayers := []Layer{}
+		for _, candidate := range r.layers {
+			if candidate.ID != id {
+				newLayers = append(newLayers, candidate)
 			}
 		}
+		r.layers = newLayers
+		if err = r.Save(); err != nil {
+			return err
+		}
 	}
 	return err
 }
 
 func (r *layerStore) Lookup(name string) (id string, err error) {
-	layer, ok := r.byname[name]
-	if !ok {
-		layer, ok = r.byid[name]
-		if !ok {
-			return "", ErrLayerUnknown
-		}
+	if layer, ok := r.lookup(name); ok {
+		return layer.ID, nil
 	}
-	return layer.ID, nil
+	return "", ErrLayerUnknown
 }
 
 func (r *layerStore) Exists(id string) bool {
-	if layer, ok := r.byname[id]; ok {
-		id = layer.ID
-	}
-	l, exists := r.byid[id]
-	return l != nil && exists
+	_, ok := r.lookup(id)
+	return ok
 }
 
 func (r *layerStore) Get(id string) (*Layer, error) {
-	if l, ok := r.byname[id]; ok {
-		return l, nil
-	}
-	if l, ok := r.byid[id]; ok {
-		return l, nil
+	if layer, ok := r.lookup(id); ok {
+		return layer, nil
 	}
 	return nil, ErrLayerUnknown
 }
@@ -641,22 +619,32 @@ func (r *layerStore) Wipe() error {
 	return nil
 }
 
-func (r *layerStore) Changes(from, to string) ([]archive.Change, error) {
-	if layer, ok := r.byname[from]; ok {
-		from = layer.ID
-	}
-	if layer, ok := r.byname[to]; ok {
-		to = layer.ID
+func (r *layerStore) findParentAndLayer(from, to string) (fromID string, toID string, fromLayer *Layer, toLayer *Layer, err error) {
+	var ok bool
+	toLayer, ok = r.lookup(to)
+	if !ok {
+		return "", "", nil, nil, ErrLayerUnknown
 	}
+	to = toLayer.ID
 	if from == "" {
-		if layer, ok := r.byid[to]; ok {
-			from = layer.Parent
+		from = toLayer.Parent
+	}
+	if from != "" {
+		fromLayer, ok = r.lookup(from)
+		if !ok {
+			fromLayer, ok = r.lookup(toLayer.Parent)
+			if !ok {
+				return "", "", nil, nil, ErrParentUnknown
+			}
 		}
+		from = fromLayer.ID
 	}
-	if to == "" {
-		return nil, ErrLayerUnknown
-	}
-	if _, ok := r.byid[to]; !ok {
+	return from, to, fromLayer, toLayer, nil
+}
+
+func (r *layerStore) Changes(from, to string) ([]archive.Change, error) {
+	from, to, _, _, err := r.findParentAndLayer(from, to)
+	if err != nil {
 		return nil, ErrLayerUnknown
 	}
 	return r.driver.Changes(to, from)
@@ -694,32 +682,19 @@ func (r *layerStore) newFileGetter(id string) (drivers.FileGetCloser, error) {
 func (r *layerStore) Diff(from, to string) (io.ReadCloser, error) {
 	var metadata storage.Unpacker
 
-	if layer, ok := r.byname[from]; ok {
-		from = layer.ID
-	}
-	if layer, ok := r.byname[to]; ok {
-		to = layer.ID
-	}
-	if from == "" {
-		if layer, ok := r.byid[to]; ok {
-			from = layer.Parent
-		}
-	}
-	if to == "" {
-		return nil, ErrParentUnknown
-	}
-	if _, ok := r.byid[to]; !ok {
+	from, to, _, toLayer, err := r.findParentAndLayer(from, to)
+	if err != nil {
 		return nil, ErrLayerUnknown
 	}
 	compression := archive.Uncompressed
-	if cflag, ok := r.byid[to].Flags[compressionFlag]; ok {
+	if cflag, ok := toLayer.Flags[compressionFlag]; ok {
 		if ctype, ok := cflag.(float64); ok {
 			compression = archive.Compression(ctype)
 		} else if ctype, ok := cflag.(archive.Compression); ok {
 			compression = archive.Compression(ctype)
 		}
 	}
-	if from != r.byid[to].Parent {
+	if from != toLayer.Parent {
 		diff, err := r.driver.Diff(to, from)
 		if err == nil && (compression != archive.Uncompressed) {
 			preader, pwriter := io.Pipe()
@@ -797,31 +772,15 @@ func (r *layerStore) Diff(from, to string) (io.ReadCloser, error) {
 }
 
 func (r *layerStore) DiffSize(from, to string) (size int64, err error) {
-	if layer, ok := r.byname[from]; ok {
-		from = layer.ID
-	}
-	if layer, ok := r.byname[to]; ok {
-		to = layer.ID
-	}
-	if from == "" {
-		if layer, ok := r.byid[to]; ok {
-			from = layer.Parent
-		}
-	}
-	if to == "" {
-		return -1, ErrParentUnknown
-	}
-	if _, ok := r.byid[to]; !ok {
+	from, to, _, _, err = r.findParentAndLayer(from, to)
+	if err != nil {
 		return -1, ErrLayerUnknown
 	}
 	return r.driver.DiffSize(to, from)
 }
 
 func (r *layerStore) ApplyDiff(to string, diff archive.Reader) (size int64, err error) {
-	if layer, ok := r.byname[to]; ok {
-		to = layer.ID
-	}
-	layer, ok := r.byid[to]
+	layer, ok := r.lookup(to)
 	if !ok {
 		return -1, ErrLayerUnknown
 	}

vendor/github.com/containers/storage/storage/lockfile.go

@@ -4,9 +4,10 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
-	"syscall"
 	"time"
 
+	"golang.org/x/sys/unix"
+
 	"github.com/containers/storage/pkg/stringid"
 )
 
@@ -51,7 +52,7 @@ func GetLockfile(path string) (Locker, error) {
 	if locker, ok := lockfiles[filepath.Clean(path)]; ok {
 		return locker, nil
 	}
-	fd, err := syscall.Open(filepath.Clean(path), os.O_RDWR|os.O_CREATE, syscall.S_IRUSR|syscall.S_IWUSR)
+	fd, err := unix.Open(filepath.Clean(path), os.O_RDWR|os.O_CREATE, unix.S_IRUSR|unix.S_IWUSR)
 	if err != nil {
 		return nil, err
 	}
@@ -61,28 +62,28 @@ func GetLockfile(path string) (Locker, error) {
 }
 
 func (l *lockfile) Lock() {
-	lk := syscall.Flock_t{
-		Type:   syscall.F_WRLCK,
+	lk := unix.Flock_t{
+		Type:   unix.F_WRLCK,
 		Whence: int16(os.SEEK_SET),
 		Start:  0,
 		Len:    0,
 		Pid:    int32(os.Getpid()),
 	}
 	l.mu.Lock()
-	for syscall.FcntlFlock(l.fd, syscall.F_SETLKW, &lk) != nil {
+	for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil {
 		time.Sleep(10 * time.Millisecond)
 	}
 }
 
 func (l *lockfile) Unlock() {
-	lk := syscall.Flock_t{
-		Type:   syscall.F_UNLCK,
+	lk := unix.Flock_t{
+		Type:   unix.F_UNLCK,
 		Whence: int16(os.SEEK_SET),
 		Start:  0,
 		Len:    0,
 		Pid:    int32(os.Getpid()),
 	}
-	for syscall.FcntlFlock(l.fd, syscall.F_SETLKW, &lk) != nil {
+	for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil {
 		time.Sleep(10 * time.Millisecond)
 	}
 	l.mu.Unlock()
@@ -91,18 +92,18 @@ func (l *lockfile) Unlock() {
 func (l *lockfile) Touch() error {
 	l.lw = stringid.GenerateRandomID()
 	id := []byte(l.lw)
-	_, err := syscall.Seek(int(l.fd), 0, os.SEEK_SET)
+	_, err := unix.Seek(int(l.fd), 0, os.SEEK_SET)
 	if err != nil {
 		return err
 	}
-	n, err := syscall.Write(int(l.fd), id)
+	n, err := unix.Write(int(l.fd), id)
 	if err != nil {
 		return err
 	}
 	if n != len(id) {
-		return syscall.ENOSPC
+		return unix.ENOSPC
 	}
-	err = syscall.Fsync(int(l.fd))
+	err = unix.Fsync(int(l.fd))
 	if err != nil {
 		return err
 	}
@@ -111,16 +112,16 @@ func (l *lockfile) Touch() error {
 
 func (l *lockfile) Modified() (bool, error) {
 	id := []byte(l.lw)
-	_, err := syscall.Seek(int(l.fd), 0, os.SEEK_SET)
+	_, err := unix.Seek(int(l.fd), 0, os.SEEK_SET)
 	if err != nil {
 		return true, err
 	}
-	n, err := syscall.Read(int(l.fd), id)
+	n, err := unix.Read(int(l.fd), id)
 	if err != nil {
 		return true, err
 	}
 	if n != len(id) {
-		return true, syscall.ENOSPC
+		return true, unix.ENOSPC
 	}
 	lw := l.lw
 	l.lw = string(id)
@@ -128,11 +129,11 @@ func (l *lockfile) Modified() (bool, error) {
 }
 
 func (l *lockfile) TouchedSince(when time.Time) bool {
-	st := syscall.Stat_t{}
-	err := syscall.Fstat(int(l.fd), &st)
+	st := unix.Stat_t{}
+	err := unix.Fstat(int(l.fd), &st)
 	if err != nil {
 		return true
 	}
-	touched := time.Unix(st.Mtim.Unix())
+	touched := time.Unix(statTMtimeUnix(st))
 	return when.Before(touched)
 }

vendor/github.com/containers/storage/storage/stat_mtim.go

@@ -0,0 +1,11 @@
+// +build linux solaris
+
+package storage
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+func statTMtimeUnix(st unix.Stat_t) (int64, int64) {
+	return st.Mtim.Unix()
+}

vendor/github.com/containers/storage/storage/stat_mtimespec.go

@@ -0,0 +1,11 @@
+// +build !linux,!solaris
+
+package storage
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+func statTMtimeUnix(st unix.Stat_t) (int64, int64) {
+	return st.Mtimespec.Unix()
+}

vendor/github.com/containers/storage/storage/store.go

@@ -2176,6 +2176,17 @@ func makeBigDataBaseName(key string) string {
 	return key
 }
 
+func stringSliceWithoutValue(slice []string, value string) []string {
+	modified := []string{}
+	for _, v := range slice {
+		if v == value {
+			continue
+		}
+		modified = append(modified, v)
+	}
+	return modified
+}
+
 func init() {
 	DefaultStoreOptions.RunRoot = "/var/run/containers/storage"
 	DefaultStoreOptions.GraphRoot = "/var/lib/containers/storage"

vendor/github.com/containers/storage/vendor.conf

@@ -10,6 +10,7 @@ github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062
 github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07
 github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d
 github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9
+github.com/tchap/go-patricia v2.2.6
 github.com/vbatts/tar-split bd4c5d64c3e9297f410025a3b1bd0c58f659e721
 github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3
 golang.org/x/net f2499483f923065a842d38eb4c7f1927e6fc6e6d