From f3207e76c878e4859018185c4fec9162d327e1e8 Mon Sep 17 00:00:00 2001 From: Patrick Devine Date: Mon, 13 Jul 2015 13:08:13 -0700 Subject: [PATCH] Catalog for V2 API Implementation This change adds a basic catalog endpoint to the API, which returns a list, or partial list, of all of the repositories contained in the registry. Calls to this endpoint are somewhat expensive, as every call requires walking a large part of the registry. Instead, to maintain a list of repositories, you would first call the catalog endpoint to get an initial list, and then use the events API to maintain any future repositories. Signed-off-by: Patrick Devine --- docs/api/v2/routes.go | 1 + docs/api/v2/urls.go | 12 +++ docs/client/repository.go | 68 +++++++++++++++++ docs/client/repository_test.go | 41 ++++++++++ docs/handlers/api_test.go | 136 +++++++++++++++++++++++++++++++++ docs/handlers/app.go | 28 ++++++- docs/handlers/catalog.go | 82 ++++++++++++++++++++ docs/handlers/context.go | 3 + docs/storage/catalog.go | 62 +++++++++++++++ docs/storage/catalog_test.go | 127 ++++++++++++++++++++++++++++++ docs/storage/registry.go | 9 +++ 11 files changed, 568 insertions(+), 1 deletion(-) create mode 100644 docs/handlers/catalog.go create mode 100644 docs/storage/catalog.go create mode 100644 docs/storage/catalog_test.go diff --git a/docs/api/v2/routes.go b/docs/api/v2/routes.go index d18860f5..5b80d5be 100644 --- a/docs/api/v2/routes.go +++ b/docs/api/v2/routes.go @@ -16,6 +16,7 @@ const ( var allEndpoints = []string{ RouteNameManifest, + RouteNameCatalog, RouteNameTags, RouteNameBlob, RouteNameBlobUpload, diff --git a/docs/api/v2/urls.go b/docs/api/v2/urls.go index 60aad565..42974394 100644 --- a/docs/api/v2/urls.go +++ b/docs/api/v2/urls.go @@ -100,6 +100,18 @@ func (ub *URLBuilder) BuildBaseURL() (string, error) { return baseURL.String(), nil } +// BuildCatalogURL constructs a url get a catalog of repositories +func (ub *URLBuilder) BuildCatalogURL(values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameCatalog) + + catalogURL, err := route.URL() + if err != nil { + return "", err + } + + return appendValuesURL(catalogURL, values...).String(), nil +} + // BuildTagsURL constructs a url to list the tags in the named repository. func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { route := ub.cloneRoute(RouteNameTags) diff --git a/docs/client/repository.go b/docs/client/repository.go index fc90cb6e..6d2fd6e7 100644 --- a/docs/client/repository.go +++ b/docs/client/repository.go @@ -444,3 +444,71 @@ func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distributi return distribution.Descriptor{}, handleErrorResponse(resp) } } + +// NewCatalog can be used to get a list of repositories +func NewCatalog(ctx context.Context, baseURL string, transport http.RoundTripper) (distribution.CatalogService, error) { + ub, err := v2.NewURLBuilderFromString(baseURL) + if err != nil { + return nil, err + } + + client := &http.Client{ + Transport: transport, + Timeout: 1 * time.Minute, + } + + return &catalog{ + client: client, + ub: ub, + context: ctx, + }, nil +} + +type catalog struct { + client *http.Client + ub *v2.URLBuilder + context context.Context +} + +func (c *catalog) Get(maxEntries int, last string) ([]string, bool, error) { + var repos []string + + values := url.Values{} + + if maxEntries > 0 { + values.Add("n", strconv.Itoa(maxEntries)) + } + + if last != "" { + values.Add("last", last) + } + + u, err := c.ub.BuildCatalogURL(values) + if err != nil { + return nil, false, err + } + + resp, err := c.client.Get(u) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + var ctlg struct { + Repositories []string `json:"repositories"` + } + decoder := json.NewDecoder(resp.Body) + + if err := decoder.Decode(&ctlg); err != nil { + return nil, false, err + } + + repos = ctlg.Repositories + default: + return nil, false, handleErrorResponse(resp) + } + + return repos, false, nil +} diff --git a/docs/client/repository_test.go b/docs/client/repository_test.go index 3a91be98..e9735cd4 100644 --- a/docs/client/repository_test.go +++ b/docs/client/repository_test.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -77,6 +78,23 @@ func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.R }) } +func addTestCatalog(content []byte, m *testutil.RequestResponseMap) { + *m = append(*m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/_catalog", + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: content, + Headers: http.Header(map[string][]string{ + "Content-Length": {strconv.Itoa(len(content))}, + "Content-Type": {"application/json; charset=utf-8"}, + }), + }, + }) +} + func TestBlobFetch(t *testing.T) { d1, b1 := newRandomBlob(1024) var m testutil.RequestResponseMap @@ -732,3 +750,26 @@ func TestManifestUnauthorized(t *testing.T) { t.Fatalf("Unexpected message value: %q, expected %q", v2Err.Message, expected) } } + +func TestCatalog(t *testing.T) { + var m testutil.RequestResponseMap + addTestCatalog([]byte("{\"repositories\":[\"foo\", \"bar\", \"baz\"]}"), &m) + + e, c := testServer(m) + defer c() + + ctx := context.Background() + ctlg, err := NewCatalog(ctx, e, nil) + if err != nil { + t.Fatal(err) + } + + repos, _, err := ctlg.Get(0, "") + if err != nil { + t.Fatal(err) + } + + if len(repos) != 3 { + t.Fatalf("Got wrong number of repos") + } +} diff --git a/docs/handlers/api_test.go b/docs/handlers/api_test.go index 8d631941..d768a116 100644 --- a/docs/handlers/api_test.go +++ b/docs/handlers/api_test.go @@ -60,6 +60,85 @@ func TestCheckAPI(t *testing.T) { } } +func TestCatalogAPI(t *testing.T) { + env := newTestEnv(t) + + values := url.Values{"last": []string{""}, "n": []string{"100"}} + + catalogURL, err := env.builder.BuildCatalogURL(values) + if err != nil { + t.Fatalf("unexpected error building catalog url: %v", err) + } + + // ----------------------------------- + // try to get an empty catalog + resp, err := http.Get(catalogURL) + if err != nil { + t.Fatalf("unexpected error issuing request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "issuing catalog api check", resp, http.StatusOK) + + var ctlg struct { + Repositories []string `json:"repositories"` + } + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&ctlg); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + // we haven't pushed anything to the registry yet + if ctlg.Repositories != nil { + t.Fatalf("repositories has unexpected values") + } + + if resp.Header.Get("Link") != "" { + t.Fatalf("repositories has more data when none expected") + } + + // ----------------------------------- + // push something to the registry and try again + imageName := "foo/bar" + createRepository(env, t, imageName, "sometag") + + resp, err = http.Get(catalogURL) + if err != nil { + t.Fatalf("unexpected error issuing request: %v", err) + } + defer resp.Body.Close() + + checkResponse(t, "issuing catalog api check", resp, http.StatusOK) + + dec = json.NewDecoder(resp.Body) + if err = dec.Decode(&ctlg); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + if len(ctlg.Repositories) != 1 { + t.Fatalf("repositories has unexpected values") + } + + if !contains(ctlg.Repositories, imageName) { + t.Fatalf("didn't find our repository '%s' in the catalog", imageName) + } + + if resp.Header.Get("Link") != "" { + t.Fatalf("repositories has more data when none expected") + } + +} + +func contains(elems []string, e string) bool { + for _, elem := range elems { + if elem == e { + return true + } + } + return false +} + func TestURLPrefix(t *testing.T) { config := configuration.Configuration{ Storage: configuration.Storage{ @@ -869,3 +948,60 @@ func checkErr(t *testing.T, err error, msg string) { t.Fatalf("unexpected error %s: %v", msg, err) } } + +func createRepository(env *testEnv, t *testing.T, imageName string, tag string) { + unsignedManifest := &manifest.Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 1, + }, + Name: imageName, + Tag: tag, + FSLayers: []manifest.FSLayer{ + { + BlobSum: "asdf", + }, + { + BlobSum: "qwer", + }, + }, + } + + // Push 2 random layers + expectedLayers := make(map[digest.Digest]io.ReadSeeker) + + for i := range unsignedManifest.FSLayers { + rs, dgstStr, err := testutil.CreateRandomTarFile() + + if err != nil { + t.Fatalf("error creating random layer %d: %v", i, err) + } + dgst := digest.Digest(dgstStr) + + expectedLayers[dgst] = rs + unsignedManifest.FSLayers[i].BlobSum = dgst + + uploadURLBase, _ := startPushLayer(t, env.builder, imageName) + pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs) + } + + signedManifest, err := manifest.Sign(unsignedManifest, env.pk) + if err != nil { + t.Fatalf("unexpected error signing manifest: %v", err) + } + + payload, err := signedManifest.Payload() + checkErr(t, err, "getting manifest payload") + + dgst, err := digest.FromBytes(payload) + checkErr(t, err, "digesting manifest") + + manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) + checkErr(t, err, "building manifest url") + + resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) + checkResponse(t, "putting signed manifest", resp, http.StatusAccepted) + checkHeaders(t, resp, http.Header{ + "Location": []string{manifestDigestURL}, + "Docker-Content-Digest": []string{dgst.String()}, + }) +} diff --git a/docs/handlers/app.go b/docs/handlers/app.go index c895222b..45f97966 100644 --- a/docs/handlers/app.go +++ b/docs/handlers/app.go @@ -69,6 +69,7 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App return http.HandlerFunc(apiBase) }) app.register(v2.RouteNameManifest, imageManifestDispatcher) + app.register(v2.RouteNameCatalog, catalogDispatcher) app.register(v2.RouteNameTags, tagsDispatcher) app.register(v2.RouteNameBlob, blobDispatcher) app.register(v2.RouteNameBlobUpload, blobUploadDispatcher) @@ -366,6 +367,9 @@ func (app *App) dispatcher(dispatch dispatchFunc) http.Handler { // Add username to request logging context.Context = ctxu.WithLogger(context.Context, ctxu.GetLogger(context.Context, "auth.user.name")) + catalog := app.registry.Catalog(context) + context.Catalog = catalog + if app.nameRequired(r) { repository, err := app.registry.Repository(context, getName(context)) @@ -493,6 +497,7 @@ func (app *App) authorized(w http.ResponseWriter, r *http.Request, context *Cont } return fmt.Errorf("forbidden: no repository name") } + accessRecords = appendCatalogAccessRecord(accessRecords, r) } ctx, err := app.accessController.Authorized(context.Context, accessRecords...) @@ -538,7 +543,8 @@ func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listene // nameRequired returns true if the route requires a name. func (app *App) nameRequired(r *http.Request) bool { route := mux.CurrentRoute(r) - return route == nil || route.GetName() != v2.RouteNameBase + routeName := route.GetName() + return route == nil || (routeName != v2.RouteNameBase && routeName != v2.RouteNameCatalog) } // apiBase implements a simple yes-man for doing overall checks against the @@ -588,6 +594,26 @@ func appendAccessRecords(records []auth.Access, method string, repo string) []au return records } +// Add the access record for the catalog if it's our current route +func appendCatalogAccessRecord(accessRecords []auth.Access, r *http.Request) []auth.Access { + route := mux.CurrentRoute(r) + routeName := route.GetName() + + if routeName == v2.RouteNameCatalog { + resource := auth.Resource{ + Type: "registry", + Name: "catalog", + } + + accessRecords = append(accessRecords, + auth.Access{ + Resource: resource, + Action: "*", + }) + } + return accessRecords +} + // applyRegistryMiddleware wraps a registry instance with the configured middlewares func applyRegistryMiddleware(registry distribution.Namespace, middlewares []configuration.Middleware) (distribution.Namespace, error) { for _, mw := range middlewares { diff --git a/docs/handlers/catalog.go b/docs/handlers/catalog.go new file mode 100644 index 00000000..fd2af76e --- /dev/null +++ b/docs/handlers/catalog.go @@ -0,0 +1,82 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/docker/distribution/registry/api/errcode" + "github.com/gorilla/handlers" +) + +const maximumReturnedEntries = 100 + +func catalogDispatcher(ctx *Context, r *http.Request) http.Handler { + catalogHandler := &catalogHandler{ + Context: ctx, + } + + return handlers.MethodHandler{ + "GET": http.HandlerFunc(catalogHandler.GetCatalog), + } +} + +type catalogHandler struct { + *Context +} + +type catalogAPIResponse struct { + Repositories []string `json:"repositories"` +} + +func (ch *catalogHandler) GetCatalog(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + lastEntry := q.Get("last") + maxEntries, err := strconv.Atoi(q.Get("n")) + if err != nil || maxEntries < 0 { + maxEntries = maximumReturnedEntries + } + + repos, moreEntries, err := ch.Catalog.Get(maxEntries, lastEntry) + if err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + + // Add a link header if there are more entries to retrieve + if moreEntries { + urlStr, err := createLinkEntry(r.URL.String(), maxEntries, repos) + if err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + w.Header().Set("Link", urlStr) + } + + enc := json.NewEncoder(w) + if err := enc.Encode(catalogAPIResponse{ + Repositories: repos, + }); err != nil { + ch.Errors = append(ch.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } +} + +// Use the original URL from the request to create a new URL for +// the link header +func createLinkEntry(origURL string, maxEntries int, repos []string) (string, error) { + calledURL, err := url.Parse(origURL) + if err != nil { + return "", err + } + + calledURL.RawQuery = fmt.Sprintf("n=%d&last=%s", maxEntries, repos[len(repos)-1]) + calledURL.Fragment = "" + urlStr := fmt.Sprintf("<%s>; rel=\"next\"", calledURL.String()) + + return urlStr, nil +} diff --git a/docs/handlers/context.go b/docs/handlers/context.go index 85a17123..6625551d 100644 --- a/docs/handlers/context.go +++ b/docs/handlers/context.go @@ -32,6 +32,9 @@ type Context struct { urlBuilder *v2.URLBuilder + // Catalog allows getting a complete list of the contents of the registry. + Catalog distribution.CatalogService + // TODO(stevvooe): The goal is too completely factor this context and // dispatching out of the web application. Ideally, we should lean on // context.Context for injection of these resources. diff --git a/docs/storage/catalog.go b/docs/storage/catalog.go new file mode 100644 index 00000000..ce184dba --- /dev/null +++ b/docs/storage/catalog.go @@ -0,0 +1,62 @@ +package storage + +import ( + "path" + "sort" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/docker/distribution" + "github.com/docker/distribution/context" + storageDriver "github.com/docker/distribution/registry/storage/driver" +) + +type catalogSvc struct { + ctx context.Context + driver storageDriver.StorageDriver +} + +var _ distribution.CatalogService = &catalogSvc{} + +// Get returns a list, or partial list, of repositories in the registry. +// Because it's a quite expensive operation, it should only be used when building up +// an initial set of repositories. +func (c *catalogSvc) Get(maxEntries int, lastEntry string) ([]string, bool, error) { + log.Infof("Retrieving up to %d entries of the catalog starting with '%s'", maxEntries, lastEntry) + var repos []string + + root, err := defaultPathMapper.path(repositoriesRootPathSpec{}) + if err != nil { + return repos, false, err + } + + Walk(c.ctx, c.driver, root, func(fileInfo storageDriver.FileInfo) error { + filePath := fileInfo.Path() + + // lop the base path off + repoPath := filePath[len(root)+1:] + + _, file := path.Split(repoPath) + if file == "_layers" { + repoPath = strings.TrimSuffix(repoPath, "/_layers") + if repoPath > lastEntry { + repos = append(repos, repoPath) + } + return ErrSkipDir + } else if strings.HasPrefix(file, "_") { + return ErrSkipDir + } + + return nil + }) + + sort.Strings(repos) + + moreEntries := false + if len(repos) > maxEntries { + moreEntries = true + repos = repos[0:maxEntries] + } + + return repos, moreEntries, nil +} diff --git a/docs/storage/catalog_test.go b/docs/storage/catalog_test.go new file mode 100644 index 00000000..8d9f3854 --- /dev/null +++ b/docs/storage/catalog_test.go @@ -0,0 +1,127 @@ +package storage + +import ( + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/cache/memory" + "github.com/docker/distribution/registry/storage/driver" + "github.com/docker/distribution/registry/storage/driver/inmemory" +) + +type setupEnv struct { + ctx context.Context + driver driver.StorageDriver + expected []string + registry distribution.Namespace + catalog distribution.CatalogService +} + +func setupFS(t *testing.T) *setupEnv { + d := inmemory.New() + c := []byte("") + ctx := context.Background() + registry := NewRegistryWithDriver(ctx, d, memory.NewInMemoryBlobDescriptorCacheProvider()) + rootpath, _ := defaultPathMapper.path(repositoriesRootPathSpec{}) + + repos := []string{ + "/foo/a/_layers/1", + "/foo/b/_layers/2", + "/bar/c/_layers/3", + "/bar/d/_layers/4", + "/foo/d/in/_layers/5", + "/an/invalid/repo", + "/bar/d/_layers/ignored/dir/6", + } + + for _, repo := range repos { + if err := d.PutContent(ctx, rootpath+repo, c); err != nil { + t.Fatalf("Unable to put to inmemory fs") + } + } + + catalog := registry.Catalog(ctx) + + expected := []string{ + "bar/c", + "bar/d", + "foo/a", + "foo/b", + "foo/d/in", + } + + return &setupEnv{ + ctx: ctx, + driver: d, + expected: expected, + registry: registry, + catalog: catalog, + } +} + +func TestCatalog(t *testing.T) { + env := setupFS(t) + + repos, more, _ := env.catalog.Get(100, "") + + if !testEq(repos, env.expected) { + t.Errorf("Expected catalog repos err") + } + + if more { + t.Errorf("Catalog has more values which we aren't expecting") + } +} + +func TestCatalogInParts(t *testing.T) { + env := setupFS(t) + + chunkLen := 2 + + repos, more, _ := env.catalog.Get(chunkLen, "") + if !testEq(repos, env.expected[0:chunkLen]) { + t.Errorf("Expected catalog first chunk err") + } + + if !more { + t.Errorf("Expected more values in catalog") + } + + lastRepo := repos[len(repos)-1] + repos, more, _ = env.catalog.Get(chunkLen, lastRepo) + + if !testEq(repos, env.expected[chunkLen:chunkLen*2]) { + t.Errorf("Expected catalog second chunk err") + } + + if !more { + t.Errorf("Expected more values in catalog") + } + + lastRepo = repos[len(repos)-1] + repos, more, _ = env.catalog.Get(chunkLen, lastRepo) + + if !testEq(repos, env.expected[chunkLen*2:chunkLen*3-1]) { + t.Errorf("Expected catalog third chunk err") + } + + if more { + t.Errorf("Catalog has more values which we aren't expecting") + } + +} + +func testEq(a, b []string) bool { + if len(a) != len(b) { + return false + } + + for count := range a { + if a[count] != b[count] { + return false + } + } + + return true +} diff --git a/docs/storage/registry.go b/docs/storage/registry.go index cf0fe3e7..17035555 100644 --- a/docs/storage/registry.go +++ b/docs/storage/registry.go @@ -55,6 +55,15 @@ func (reg *registry) Scope() distribution.Scope { return distribution.GlobalScope } +// Catalog returns an instance of the catalog service which can be +// used to dump all of the repositories in a registry +func (reg *registry) Catalog(ctx context.Context) distribution.CatalogService { + return &catalogSvc{ + ctx: ctx, + driver: reg.blobStore.driver, + } +} + // Repository returns an instance of the repository tied to the registry. // Instances should not be shared between goroutines but are cheap to // allocate. In general, they should be request scoped.