Catalog for V2 API Implementation

This change adds a basic catalog endpoint to the API, which returns a list,
or partial list, of all of the repositories contained in the registry.  Calls
to this endpoint are somewhat expensive, as every call requires walking a
large part of the registry.

Instead, to maintain a list of repositories, you would first call the catalog
endpoint to get an initial list, and then use the events API to maintain
any future repositories.

Signed-off-by: Patrick Devine <patrick.devine@docker.com>
This commit is contained in:
Patrick Devine 2015-07-13 13:08:13 -07:00
parent 006214d902
commit 74563efe98
12 changed files with 577 additions and 1 deletions

View file

@ -35,6 +35,9 @@ type Namespace interface {
// registry may or may not have the repository but should always return a // registry may or may not have the repository but should always return a
// reference. // reference.
Repository(ctx context.Context, name string) (Repository, error) Repository(ctx context.Context, name string) (Repository, error)
// Catalog returns a reference which can be used for listing repositories
Catalog(ctx context.Context) CatalogService
} }
// ManifestServiceOption is a function argument for Manifest Service methods // ManifestServiceOption is a function argument for Manifest Service methods
@ -112,3 +115,9 @@ type SignatureService interface {
// Put stores the signature for the provided digest. // Put stores the signature for the provided digest.
Put(dgst digest.Digest, signatures ...[]byte) error Put(dgst digest.Digest, signatures ...[]byte) error
} }
// CatalogService provides a way of retrieving the names of each of the repositories
type CatalogService interface {
// Get retrieves repository names from the registry.
Get(n int, q string) (p []string, moreEntries bool, err error)
}

View file

@ -16,6 +16,7 @@ const (
var allEndpoints = []string{ var allEndpoints = []string{
RouteNameManifest, RouteNameManifest,
RouteNameCatalog,
RouteNameTags, RouteNameTags,
RouteNameBlob, RouteNameBlob,
RouteNameBlobUpload, RouteNameBlobUpload,

View file

@ -100,6 +100,18 @@ func (ub *URLBuilder) BuildBaseURL() (string, error) {
return baseURL.String(), nil return baseURL.String(), nil
} }
// BuildCatalogURL constructs a url get a catalog of repositories
func (ub *URLBuilder) BuildCatalogURL(values ...url.Values) (string, error) {
route := ub.cloneRoute(RouteNameCatalog)
catalogURL, err := route.URL()
if err != nil {
return "", err
}
return appendValuesURL(catalogURL, values...).String(), nil
}
// BuildTagsURL constructs a url to list the tags in the named repository. // BuildTagsURL constructs a url to list the tags in the named repository.
func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { func (ub *URLBuilder) BuildTagsURL(name string) (string, error) {
route := ub.cloneRoute(RouteNameTags) route := ub.cloneRoute(RouteNameTags)

View file

@ -444,3 +444,71 @@ func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distributi
return distribution.Descriptor{}, handleErrorResponse(resp) return distribution.Descriptor{}, handleErrorResponse(resp)
} }
} }
// NewCatalog can be used to get a list of repositories
func NewCatalog(ctx context.Context, baseURL string, transport http.RoundTripper) (distribution.CatalogService, error) {
ub, err := v2.NewURLBuilderFromString(baseURL)
if err != nil {
return nil, err
}
client := &http.Client{
Transport: transport,
Timeout: 1 * time.Minute,
}
return &catalog{
client: client,
ub: ub,
context: ctx,
}, nil
}
type catalog struct {
client *http.Client
ub *v2.URLBuilder
context context.Context
}
func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) {
var repos []string
values := url.Values{}
if maxEntries > 0 {
values.Add("n", strconv.Itoa(maxEntries))
}
if last != "" {
values.Add("last", last)
}
u, err := c.ub.BuildCatalogURL(values)
if err != nil {
return nil, false, err
}
resp, err := c.client.Get(u)
if err != nil {
return nil, false, err
}
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
var ctlg struct {
Repositories []string `json:"repositories"`
}
decoder := json.NewDecoder(resp.Body)
if err := decoder.Decode(&ctlg); err != nil {
return nil, false, err
}
repos = ctlg.Repositories
default:
return nil, false, handleErrorResponse(resp)
}
return repos, false, nil
}

View file

@ -8,6 +8,7 @@ import (
"log" "log"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -77,6 +78,23 @@ func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.R
}) })
} }
func addTestCatalog(content []byte, m *testutil.RequestResponseMap) {
*m = append(*m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/_catalog",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: content,
Headers: http.Header(map[string][]string{
"Content-Length": {strconv.Itoa(len(content))},
"Content-Type": {"application/json; charset=utf-8"},
}),
},
})
}
func TestBlobFetch(t *testing.T) { func TestBlobFetch(t *testing.T) {
d1, b1 := newRandomBlob(1024) d1, b1 := newRandomBlob(1024)
var m testutil.RequestResponseMap var m testutil.RequestResponseMap
@ -732,3 +750,26 @@ func TestManifestUnauthorized(t *testing.T) {
t.Fatalf("Unexpected message value: %q, expected %q", v2Err.Message, expected) t.Fatalf("Unexpected message value: %q, expected %q", v2Err.Message, expected)
} }
} }
func TestCatalog(t *testing.T) {
var m testutil.RequestResponseMap
addTestCatalog([]byte("{\"repositories\":[\"foo\", \"bar\", \"baz\"]}"), &m)
e, c := testServer(m)
defer c()
ctx := context.Background()
ctlg, err := NewCatalog(ctx, e, nil)
if err != nil {
t.Fatal(err)
}
repos, _, err := ctlg.Get(0, "")
if err != nil {
t.Fatal(err)
}
if len(repos) != 3 {
t.Fatalf("Got wrong number of repos")
}
}

View file

@ -60,6 +60,85 @@ func TestCheckAPI(t *testing.T) {
} }
} }
func TestCatalogAPI(t *testing.T) {
env := newTestEnv(t)
values := url.Values{"last": []string{""}, "n": []string{"100"}}
catalogURL, err := env.builder.BuildCatalogURL(values)
if err != nil {
t.Fatalf("unexpected error building catalog url: %v", err)
}
// -----------------------------------
// try to get an empty catalog
resp, err := http.Get(catalogURL)
if err != nil {
t.Fatalf("unexpected error issuing request: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "issuing catalog api check", resp, http.StatusOK)
var ctlg struct {
Repositories []string `json:"repositories"`
}
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&ctlg); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
// we haven't pushed anything to the registry yet
if ctlg.Repositories != nil {
t.Fatalf("repositories has unexpected values")
}
if resp.Header.Get("Link") != "" {
t.Fatalf("repositories has more data when none expected")
}
// -----------------------------------
// push something to the registry and try again
imageName := "foo/bar"
createRepository(env, t, imageName, "sometag")
resp, err = http.Get(catalogURL)
if err != nil {
t.Fatalf("unexpected error issuing request: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "issuing catalog api check", resp, http.StatusOK)
dec = json.NewDecoder(resp.Body)
if err = dec.Decode(&ctlg); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if len(ctlg.Repositories) != 1 {
t.Fatalf("repositories has unexpected values")
}
if !contains(ctlg.Repositories, imageName) {
t.Fatalf("didn't find our repository '%s' in the catalog", imageName)
}
if resp.Header.Get("Link") != "" {
t.Fatalf("repositories has more data when none expected")
}
}
func contains(elems []string, e string) bool {
for _, elem := range elems {
if elem == e {
return true
}
}
return false
}
func TestURLPrefix(t *testing.T) { func TestURLPrefix(t *testing.T) {
config := configuration.Configuration{ config := configuration.Configuration{
Storage: configuration.Storage{ Storage: configuration.Storage{
@ -869,3 +948,60 @@ func checkErr(t *testing.T, err error, msg string) {
t.Fatalf("unexpected error %s: %v", msg, err) t.Fatalf("unexpected error %s: %v", msg, err)
} }
} }
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) {
unsignedManifest := &manifest.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: imageName,
Tag: tag,
FSLayers: []manifest.FSLayer{
{
BlobSum: "asdf",
},
{
BlobSum: "qwer",
},
},
}
// Push 2 random layers
expectedLayers := make(map[digest.Digest]io.ReadSeeker)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
dgst := digest.Digest(dgstStr)
expectedLayers[dgst] = rs
unsignedManifest.FSLayers[i].BlobSum = dgst
uploadURLBase, _ := startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs)
}
signedManifest, err := manifest.Sign(unsignedManifest, env.pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst, err := digest.FromBytes(payload)
checkErr(t, err, "digesting manifest")
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusAccepted)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
})
}

View file

@ -69,6 +69,7 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App
return http.HandlerFunc(apiBase) return http.HandlerFunc(apiBase)
}) })
app.register(v2.RouteNameManifest, imageManifestDispatcher) app.register(v2.RouteNameManifest, imageManifestDispatcher)
app.register(v2.RouteNameCatalog, catalogDispatcher)
app.register(v2.RouteNameTags, tagsDispatcher) app.register(v2.RouteNameTags, tagsDispatcher)
app.register(v2.RouteNameBlob, blobDispatcher) app.register(v2.RouteNameBlob, blobDispatcher)
app.register(v2.RouteNameBlobUpload, blobUploadDispatcher) app.register(v2.RouteNameBlobUpload, blobUploadDispatcher)
@ -366,6 +367,9 @@ func (app *App) dispatcher(dispatch dispatchFunc) http.Handler {
// Add username to request logging // Add username to request logging
context.Context = ctxu.WithLogger(context.Context, ctxu.GetLogger(context.Context, "auth.user.name")) context.Context = ctxu.WithLogger(context.Context, ctxu.GetLogger(context.Context, "auth.user.name"))
catalog := app.registry.Catalog(context)
context.Catalog = catalog
if app.nameRequired(r) { if app.nameRequired(r) {
repository, err := app.registry.Repository(context, getName(context)) repository, err := app.registry.Repository(context, getName(context))
@ -493,6 +497,7 @@ func (app *App) authorized(w http.ResponseWriter, r *http.Request, context *Cont
} }
return fmt.Errorf("forbidden: no repository name") return fmt.Errorf("forbidden: no repository name")
} }
accessRecords = appendCatalogAccessRecord(accessRecords, r)
} }
ctx, err := app.accessController.Authorized(context.Context, accessRecords...) ctx, err := app.accessController.Authorized(context.Context, accessRecords...)
@ -538,7 +543,8 @@ func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listene
// nameRequired returns true if the route requires a name. // nameRequired returns true if the route requires a name.
func (app *App) nameRequired(r *http.Request) bool { func (app *App) nameRequired(r *http.Request) bool {
route := mux.CurrentRoute(r) route := mux.CurrentRoute(r)
return route == nil || route.GetName() != v2.RouteNameBase routeName := route.GetName()
return route == nil || (routeName != v2.RouteNameBase && routeName != v2.RouteNameCatalog)
} }
// apiBase implements a simple yes-man for doing overall checks against the // apiBase implements a simple yes-man for doing overall checks against the
@ -588,6 +594,26 @@ func appendAccessRecords(records []auth.Access, method string, repo string) []au
return records return records
} }
// Add the access record for the catalog if it's our current route
func appendCatalogAccessRecord(accessRecords []auth.Access, r *http.Request) []auth.Access {
route := mux.CurrentRoute(r)
routeName := route.GetName()
if routeName == v2.RouteNameCatalog {
resource := auth.Resource{
Type: "registry",
Name: "catalog",
}
accessRecords = append(accessRecords,
auth.Access{
Resource: resource,
Action: "*",
})
}
return accessRecords
}
// applyRegistryMiddleware wraps a registry instance with the configured middlewares // applyRegistryMiddleware wraps a registry instance with the configured middlewares
func applyRegistryMiddleware(registry distribution.Namespace, middlewares []configuration.Middleware) (distribution.Namespace, error) { func applyRegistryMiddleware(registry distribution.Namespace, middlewares []configuration.Middleware) (distribution.Namespace, error) {
for _, mw := range middlewares { for _, mw := range middlewares {

View file

@ -0,0 +1,82 @@
package handlers
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"github.com/docker/distribution/registry/api/errcode"
"github.com/gorilla/handlers"
)
const maximumReturnedEntries = 100
func catalogDispatcher(ctx *Context, r *http.Request) http.Handler {
catalogHandler := &catalogHandler{
Context: ctx,
}
return handlers.MethodHandler{
"GET": http.HandlerFunc(catalogHandler.GetCatalog),
}
}
type catalogHandler struct {
*Context
}
type catalogAPIResponse struct {
Repositories []string `json:"repositories"`
}
func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
lastEntry := q.Get("last")
maxEntries, err := strconv.Atoi(q.Get("n"))
if err != nil || maxEntries < 0 {
maxEntries = maximumReturnedEntries
}
repos, moreEntries, err := ch.Catalog.Get(maxEntries, lastEntry)
if err != nil {
ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
// Add a link header if there are more entries to retrieve
if moreEntries {
urlStr, err := createLinkEntry(r.URL.String(), maxEntries, repos)
if err != nil {
ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
w.Header().Set("Link", urlStr)
}
enc := json.NewEncoder(w)
if err := enc.Encode(catalogAPIResponse{
Repositories: repos,
}); err != nil {
ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// Use the original URL from the request to create a new URL for
// the link header
func createLinkEntry(origURL string, maxEntries int, repos []string) (string, error) {
calledURL, err := url.Parse(origURL)
if err != nil {
return "", err
}
calledURL.RawQuery = fmt.Sprintf("n=%d&last=%s", maxEntries, repos[len(repos)-1])
calledURL.Fragment = ""
urlStr := fmt.Sprintf("<%s>; rel=\"next\"", calledURL.String())
return urlStr, nil
}

View file

@ -32,6 +32,9 @@ type Context struct {
urlBuilder *v2.URLBuilder urlBuilder *v2.URLBuilder
// Catalog allows getting a complete list of the contents of the registry.
Catalog distribution.CatalogService
// TODO(stevvooe): The goal is too completely factor this context and // TODO(stevvooe): The goal is too completely factor this context and
// dispatching out of the web application. Ideally, we should lean on // dispatching out of the web application. Ideally, we should lean on
// context.Context for injection of these resources. // context.Context for injection of these resources.

View file

@ -0,0 +1,62 @@
package storage
import (
"path"
"sort"
"strings"
log "github.com/Sirupsen/logrus"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
storageDriver "github.com/docker/distribution/registry/storage/driver"
)
type catalogSvc struct {
ctx context.Context
driver storageDriver.StorageDriver
}
var _ distribution.CatalogService = &catalogSvc{}
// Get returns a list, or partial list, of repositories in the registry.
// Because it's a quite expensive operation, it should only be used when building up
// an initial set of repositories.
func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, error) {
log.Infof("Retrieving up to %d entries of the catalog starting with '%s'", maxEntries, lastEntry)
var repos []string
root, err := defaultPathMapper.path(repositoriesRootPathSpec{})
if err != nil {
return repos, false, err
}
Walk(c.ctx, c.driver, root, func(fileInfo storageDriver.FileInfo) error {
filePath := fileInfo.Path()
// lop the base path off
repoPath := filePath[len(root)+1:]
_, file := path.Split(repoPath)
if file == "_layers" {
repoPath = strings.TrimSuffix(repoPath, "/_layers")
if repoPath > lastEntry {
repos = append(repos, repoPath)
}
return ErrSkipDir
} else if strings.HasPrefix(file, "_") {
return ErrSkipDir
}
return nil
})
sort.Strings(repos)
moreEntries := false
if len(repos) > maxEntries {
moreEntries = true
repos = repos[0:maxEntries]
}
return repos, moreEntries, nil
}

View file

@ -0,0 +1,127 @@
package storage
import (
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/storage/cache/memory"
"github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/inmemory"
)
type setupEnv struct {
ctx context.Context
driver driver.StorageDriver
expected []string
registry distribution.Namespace
catalog distribution.CatalogService
}
func setupFS(t *testing.T) *setupEnv {
d := inmemory.New()
c := []byte("")
ctx := context.Background()
registry := NewRegistryWithDriver(ctx, d, memory.NewInMemoryBlobDescriptorCacheProvider())
rootpath, _ := defaultPathMapper.path(repositoriesRootPathSpec{})
repos := []string{
"/foo/a/_layers/1",
"/foo/b/_layers/2",
"/bar/c/_layers/3",
"/bar/d/_layers/4",
"/foo/d/in/_layers/5",
"/an/invalid/repo",
"/bar/d/_layers/ignored/dir/6",
}
for _, repo := range repos {
if err := d.PutContent(ctx, rootpath+repo, c); err != nil {
t.Fatalf("Unable to put to inmemory fs")
}
}
catalog := registry.Catalog(ctx)
expected := []string{
"bar/c",
"bar/d",
"foo/a",
"foo/b",
"foo/d/in",
}
return &setupEnv{
ctx: ctx,
driver: d,
expected: expected,
registry: registry,
catalog: catalog,
}
}
func TestCatalog(t *testing.T) {
env := setupFS(t)
repos, more, _ := env.catalog.Get(100, "")
if !testEq(repos, env.expected) {
t.Errorf("Expected catalog repos err")
}
if more {
t.Errorf("Catalog has more values which we aren't expecting")
}
}
func TestCatalogInParts(t *testing.T) {
env := setupFS(t)
chunkLen := 2
repos, more, _ := env.catalog.Get(chunkLen, "")
if !testEq(repos, env.expected[0:chunkLen]) {
t.Errorf("Expected catalog first chunk err")
}
if !more {
t.Errorf("Expected more values in catalog")
}
lastRepo := repos[len(repos)-1]
repos, more, _ = env.catalog.Get(chunkLen, lastRepo)
if !testEq(repos, env.expected[chunkLen:chunkLen*2]) {
t.Errorf("Expected catalog second chunk err")
}
if !more {
t.Errorf("Expected more values in catalog")
}
lastRepo = repos[len(repos)-1]
repos, more, _ = env.catalog.Get(chunkLen, lastRepo)
if !testEq(repos, env.expected[chunkLen*2:chunkLen*3-1]) {
t.Errorf("Expected catalog third chunk err")
}
if more {
t.Errorf("Catalog has more values which we aren't expecting")
}
}
func testEq(a, b []string) bool {
if len(a) != len(b) {
return false
}
for count := range a {
if a[count] != b[count] {
return false
}
}
return true
}

View file

@ -55,6 +55,15 @@ func (reg *registry) Scope() distribution.Scope {
return distribution.GlobalScope return distribution.GlobalScope
} }
// Catalog returns an instance of the catalog service which can be
// used to dump all of the repositories in a registry
func (reg *registry) Catalog(ctx context.Context) distribution.CatalogService {
return &catalogSvc{
ctx: ctx,
driver: reg.blobStore.driver,
}
}
// Repository returns an instance of the repository tied to the registry. // Repository returns an instance of the repository tied to the registry.
// Instances should not be shared between goroutines but are cheap to // Instances should not be shared between goroutines but are cheap to
// allocate. In general, they should be request scoped. // allocate. In general, they should be request scoped.