Catalog for V2 API Implementation

This change adds a basic catalog endpoint to the API, which returns a list,
or partial list, of all of the repositories contained in the registry.  Calls
to this endpoint are somewhat expensive, as every call requires walking a
large part of the registry.

Instead, to maintain a list of repositories, you would first call the catalog
endpoint to get an initial list, and then use the events API to maintain
any future repositories.

Signed-off-by: Patrick Devine <patrick.devine@docker.com>

docs/storage/catalog.go

@@ -0,0 +1,62 @@
+package storage
+
+import (
+	"path"
+	"sort"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	storageDriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+type catalogSvc struct {
+	ctx    context.Context
+	driver storageDriver.StorageDriver
+}
+
+var _ distribution.CatalogService = &catalogSvc{}
+
+// Get returns a list, or partial list, of repositories in the registry.
+// Because it's a quite expensive operation, it should only be used when building up
+// an initial set of repositories.
+func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, error) {
+	log.Infof("Retrieving up to %d entries of the catalog starting with '%s'", maxEntries, lastEntry)
+	var repos []string
+
+	root, err := defaultPathMapper.path(repositoriesRootPathSpec{})
+	if err != nil {
+		return repos, false, err
+	}
+
+	Walk(c.ctx, c.driver, root, func(fileInfo storageDriver.FileInfo) error {
+		filePath := fileInfo.Path()
+
+		// lop the base path off
+		repoPath := filePath[len(root)+1:]
+
+		_, file := path.Split(repoPath)
+		if file == "_layers" {
+			repoPath = strings.TrimSuffix(repoPath, "/_layers")
+			if repoPath > lastEntry {
+				repos = append(repos, repoPath)
+			}
+			return ErrSkipDir
+		} else if strings.HasPrefix(file, "_") {
+			return ErrSkipDir
+		}
+
+		return nil
+	})
+
+	sort.Strings(repos)
+
+	moreEntries := false
+	if len(repos) > maxEntries {
+		moreEntries = true
+		repos = repos[0:maxEntries]
+	}
+
+	return repos, moreEntries, nil
+}

docs/storage/catalog_test.go

@@ -0,0 +1,127 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/cache/memory"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+)
+
+type setupEnv struct {
+	ctx      context.Context
+	driver   driver.StorageDriver
+	expected []string
+	registry distribution.Namespace
+	catalog  distribution.CatalogService
+}
+
+func setupFS(t *testing.T) *setupEnv {
+	d := inmemory.New()
+	c := []byte("")
+	ctx := context.Background()
+	registry := NewRegistryWithDriver(ctx, d, memory.NewInMemoryBlobDescriptorCacheProvider())
+	rootpath, _ := defaultPathMapper.path(repositoriesRootPathSpec{})
+
+	repos := []string{
+		"/foo/a/_layers/1",
+		"/foo/b/_layers/2",
+		"/bar/c/_layers/3",
+		"/bar/d/_layers/4",
+		"/foo/d/in/_layers/5",
+		"/an/invalid/repo",
+		"/bar/d/_layers/ignored/dir/6",
+	}
+
+	for _, repo := range repos {
+		if err := d.PutContent(ctx, rootpath+repo, c); err != nil {
+			t.Fatalf("Unable to put to inmemory fs")
+		}
+	}
+
+	catalog := registry.Catalog(ctx)
+
+	expected := []string{
+		"bar/c",
+		"bar/d",
+		"foo/a",
+		"foo/b",
+		"foo/d/in",
+	}
+
+	return &setupEnv{
+		ctx:      ctx,
+		driver:   d,
+		expected: expected,
+		registry: registry,
+		catalog:  catalog,
+	}
+}
+
+func TestCatalog(t *testing.T) {
+	env := setupFS(t)
+
+	repos, more, _ := env.catalog.Get(100, "")
+
+	if !testEq(repos, env.expected) {
+		t.Errorf("Expected catalog repos err")
+	}
+
+	if more {
+		t.Errorf("Catalog has more values which we aren't expecting")
+	}
+}
+
+func TestCatalogInParts(t *testing.T) {
+	env := setupFS(t)
+
+	chunkLen := 2
+
+	repos, more, _ := env.catalog.Get(chunkLen, "")
+	if !testEq(repos, env.expected[0:chunkLen]) {
+		t.Errorf("Expected catalog first chunk err")
+	}
+
+	if !more {
+		t.Errorf("Expected more values in catalog")
+	}
+
+	lastRepo := repos[len(repos)-1]
+	repos, more, _ = env.catalog.Get(chunkLen, lastRepo)
+
+	if !testEq(repos, env.expected[chunkLen:chunkLen*2]) {
+		t.Errorf("Expected catalog second chunk err")
+	}
+
+	if !more {
+		t.Errorf("Expected more values in catalog")
+	}
+
+	lastRepo = repos[len(repos)-1]
+	repos, more, _ = env.catalog.Get(chunkLen, lastRepo)
+
+	if !testEq(repos, env.expected[chunkLen*2:chunkLen*3-1]) {
+		t.Errorf("Expected catalog third chunk err")
+	}
+
+	if more {
+		t.Errorf("Catalog has more values which we aren't expecting")
+	}
+
+}
+
+func testEq(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for count := range a {
+		if a[count] != b[count] {
+			return false
+		}
+	}
+
+	return true
+}

docs/storage/registry.go

@@ -55,6 +55,15 @@ func (reg *registry) Scope() distribution.Scope {
 	return distribution.GlobalScope
 }
 
+// Catalog returns an instance of the catalog service which can be
+// used to dump all of the repositories in a registry
+func (reg *registry) Catalog(ctx context.Context) distribution.CatalogService {
+	return &catalogSvc{
+		ctx:    ctx,
+		driver: reg.blobStore.driver,
+	}
+}
+
 // Repository returns an instance of the repository tied to the registry.
 // Instances should not be shared between goroutines but are cheap to
 // allocate. In general, they should be request scoped.