diff --git a/docs/api/v2/descriptors.go b/docs/api/v2/descriptors.go index 7eba362a..52c725dc 100644 --- a/docs/api/v2/descriptors.go +++ b/docs/api/v2/descriptors.go @@ -495,7 +495,7 @@ var routeDescriptors = []RouteDescriptor{ Methods: []MethodDescriptor{ { Method: "GET", - Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest.", + Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data.", Requests: []RequestDescriptor{ { Headers: []ParameterDescriptor{ diff --git a/docs/client/repository.go b/docs/client/repository.go index 8f525f0d..9d489dd5 100644 --- a/docs/client/repository.go +++ b/docs/client/repository.go @@ -3,6 +3,7 @@ package client import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -14,7 +15,6 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/client/transport" @@ -156,26 +156,139 @@ func (r *repository) Manifests(ctx context.Context, options ...distribution.Mani }, nil } -func (r *repository) Signatures() distribution.SignatureService { - ms, _ := r.Manifests(r.context) - return &signatures{ - manifests: ms, +func (r *repository) Tags(ctx context.Context) distribution.TagService { + return &tags{ + client: r.client, + ub: r.ub, + context: r.context, + name: r.Name(), } } -type signatures struct { - manifests distribution.ManifestService +// tags implements remote tagging operations. +type tags struct { + client *http.Client + ub *v2.URLBuilder + context context.Context + name string } -func (s *signatures) Get(dgst digest.Digest) ([][]byte, error) { - m, err := s.manifests.Get(dgst) +// All returns all tags +func (t *tags) All(ctx context.Context) ([]string, error) { + var tags []string + + u, err := t.ub.BuildTagsURL(t.name) if err != nil { - return nil, err + return tags, err } - return m.Signatures() + + resp, err := t.client.Get(u) + if err != nil { + return tags, err + } + defer resp.Body.Close() + + if SuccessStatus(resp.StatusCode) { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return tags, err + } + + tagsResponse := struct { + Tags []string `json:"tags"` + }{} + if err := json.Unmarshal(b, &tagsResponse); err != nil { + return tags, err + } + tags = tagsResponse.Tags + return tags, nil + } + return tags, handleErrorResponse(resp) } -func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error { +func descriptorFromResponse(response *http.Response) (distribution.Descriptor, error) { + desc := distribution.Descriptor{} + headers := response.Header + + ctHeader := headers.Get("Content-Type") + if ctHeader == "" { + return distribution.Descriptor{}, errors.New("missing or empty Content-Type header") + } + desc.MediaType = ctHeader + + digestHeader := headers.Get("Docker-Content-Digest") + if digestHeader == "" { + bytes, err := ioutil.ReadAll(response.Body) + if err != nil { + return distribution.Descriptor{}, err + } + _, desc, err := distribution.UnmarshalManifest(ctHeader, bytes) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil + } + + dgst, err := digest.ParseDigest(digestHeader) + if err != nil { + return distribution.Descriptor{}, err + } + desc.Digest = dgst + + lengthHeader := headers.Get("Content-Length") + if lengthHeader == "" { + return distribution.Descriptor{}, errors.New("missing or empty Content-Length header") + } + length, err := strconv.ParseInt(lengthHeader, 10, 64) + if err != nil { + return distribution.Descriptor{}, err + } + desc.Size = length + + return desc, nil + +} + +// Get issues a HEAD request for a Manifest against its named endpoint in order +// to construct a descriptor for the tag. If the registry doesn't support HEADing +// a manifest, fallback to GET. +func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + u, err := t.ub.BuildManifestURL(t.name, tag) + if err != nil { + return distribution.Descriptor{}, err + } + var attempts int + resp, err := t.client.Head(u) + +check: + if err != nil { + return distribution.Descriptor{}, err + } + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 400: + return descriptorFromResponse(resp) + case resp.StatusCode == http.StatusMethodNotAllowed: + resp, err = t.client.Get(u) + attempts++ + if attempts > 1 { + return distribution.Descriptor{}, err + } + goto check + default: + return distribution.Descriptor{}, handleErrorResponse(resp) + } +} + +func (t *tags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + panic("not implemented") +} + +func (t *tags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + panic("not implemented") +} + +func (t *tags) Untag(ctx context.Context, tag string) error { panic("not implemented") } @@ -186,44 +299,8 @@ type manifests struct { etags map[string]string } -func (ms *manifests) Tags() ([]string, error) { - u, err := ms.ub.BuildTagsURL(ms.name) - if err != nil { - return nil, err - } - - resp, err := ms.client.Get(u) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if SuccessStatus(resp.StatusCode) { - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - tagsResponse := struct { - Tags []string `json:"tags"` - }{} - if err := json.Unmarshal(b, &tagsResponse); err != nil { - return nil, err - } - - return tagsResponse.Tags, nil - } - return nil, handleErrorResponse(resp) -} - -func (ms *manifests) Exists(dgst digest.Digest) (bool, error) { - // Call by Tag endpoint since the API uses the same - // URL endpoint for tags and digests. - return ms.ExistsByTag(dgst.String()) -} - -func (ms *manifests) ExistsByTag(tag string) (bool, error) { - u, err := ms.ub.BuildManifestURL(ms.name, tag) +func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + u, err := ms.ub.BuildManifestURL(ms.name, dgst.String()) if err != nil { return false, err } @@ -241,46 +318,63 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) { return false, handleErrorResponse(resp) } -func (ms *manifests) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { - // Call by Tag endpoint since the API uses the same - // URL endpoint for tags and digests. - return ms.GetByTag(dgst.String()) -} - -// AddEtagToTag allows a client to supply an eTag to GetByTag which will be +// AddEtagToTag allows a client to supply an eTag to Get which will be // used for a conditional HTTP request. If the eTag matches, a nil manifest -// and nil error will be returned. etag is automatically quoted when added to -// this map. +// and ErrManifestNotModified error will be returned. etag is automatically +// quoted when added to this map. func AddEtagToTag(tag, etag string) distribution.ManifestServiceOption { - return func(ms distribution.ManifestService) error { - if ms, ok := ms.(*manifests); ok { - ms.etags[tag] = fmt.Sprintf(`"%s"`, etag) - return nil - } - return fmt.Errorf("etag options is a client-only option") - } + return etagOption{tag, etag} } -func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { +type etagOption struct{ tag, etag string } + +func (o etagOption) Apply(ms distribution.ManifestService) error { + if ms, ok := ms.(*manifests); ok { + ms.etags[o.tag] = fmt.Sprintf(`"%s"`, o.etag) + return nil + } + return fmt.Errorf("etag options is a client-only option") +} + +func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + + var tag string for _, option := range options { - err := option(ms) - if err != nil { - return nil, err + if opt, ok := option.(withTagOption); ok { + tag = opt.tag + } else { + err := option.Apply(ms) + if err != nil { + return nil, err + } } } - u, err := ms.ub.BuildManifestURL(ms.name, tag) + var ref string + if tag != "" { + ref = tag + } else { + ref = dgst.String() + } + + u, err := ms.ub.BuildManifestURL(ms.name, ref) if err != nil { return nil, err } + req, err := http.NewRequest("GET", u, nil) if err != nil { return nil, err } - if _, ok := ms.etags[tag]; ok { - req.Header.Set("If-None-Match", ms.etags[tag]) + for _, t := range distribution.ManifestMediaTypes() { + req.Header.Add("Accept", t) } + + if _, ok := ms.etags[ref]; ok { + req.Header.Set("If-None-Match", ms.etags[ref]) + } + resp, err := ms.client.Do(req) if err != nil { return nil, err @@ -289,44 +383,89 @@ func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServic if resp.StatusCode == http.StatusNotModified { return nil, distribution.ErrManifestNotModified } else if SuccessStatus(resp.StatusCode) { - var sm schema1.SignedManifest - decoder := json.NewDecoder(resp.Body) + mt := resp.Header.Get("Content-Type") + body, err := ioutil.ReadAll(resp.Body) - if err := decoder.Decode(&sm); err != nil { + if err != nil { return nil, err } - return &sm, nil + m, _, err := distribution.UnmarshalManifest(mt, body) + if err != nil { + return nil, err + } + return m, nil } return nil, handleErrorResponse(resp) } -func (ms *manifests) Put(m *schema1.SignedManifest) error { - manifestURL, err := ms.ub.BuildManifestURL(ms.name, m.Tag) - if err != nil { - return err +// WithTag allows a tag to be passed into Put which enables the client +// to build a correct URL. +func WithTag(tag string) distribution.ManifestServiceOption { + return withTagOption{tag} +} + +type withTagOption struct{ tag string } + +func (o withTagOption) Apply(m distribution.ManifestService) error { + if _, ok := m.(*manifests); ok { + return nil + } + return fmt.Errorf("withTagOption is a client-only option") +} + +// Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the +// tag name in order to build the correct upload URL. This state is written and read under a lock. +func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + var tag string + + for _, option := range options { + if opt, ok := option.(withTagOption); ok { + tag = opt.tag + } else { + err := option.Apply(ms) + if err != nil { + return "", err + } + } } - // todo(richardscothern): do something with options here when they become applicable - - putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(m.Raw)) + manifestURL, err := ms.ub.BuildManifestURL(ms.name, tag) if err != nil { - return err + return "", err } + mediaType, p, err := m.Payload() + if err != nil { + return "", err + } + + putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(p)) + if err != nil { + return "", err + } + + putRequest.Header.Set("Content-Type", mediaType) + resp, err := ms.client.Do(putRequest) if err != nil { - return err + return "", err } defer resp.Body.Close() if SuccessStatus(resp.StatusCode) { - // TODO(dmcgowan): make use of digest header - return nil + dgstHeader := resp.Header.Get("Docker-Content-Digest") + dgst, err := digest.ParseDigest(dgstHeader) + if err != nil { + return "", err + } + + return dgst, nil } - return handleErrorResponse(resp) + + return "", handleErrorResponse(resp) } -func (ms *manifests) Delete(dgst digest.Digest) error { +func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error { u, err := ms.ub.BuildManifestURL(ms.name, dgst.String()) if err != nil { return err @@ -348,6 +487,11 @@ func (ms *manifests) Delete(dgst digest.Digest) error { return handleErrorResponse(resp) } +// todo(richardscothern): Restore interface and implementation with merge of #1050 +/*func (ms *manifests) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + panic("not supported") +}*/ + type blobs struct { name string ub *v2.URLBuilder diff --git a/docs/client/repository_test.go b/docs/client/repository_test.go index a001b62f..c1032ec1 100644 --- a/docs/client/repository_test.go +++ b/docs/client/repository_test.go @@ -42,7 +42,6 @@ func newRandomBlob(size int) (digest.Digest, []byte) { } func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.RequestResponseMap) { - *m = append(*m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", @@ -499,12 +498,7 @@ func newRandomSchemaV1Manifest(name, tag string, blobCount int) (*schema1.Signed panic(err) } - p, err := sm.Payload() - if err != nil { - panic(err) - } - - return sm, digest.FromBytes(p), p + return sm, digest.FromBytes(sm.Canonical), sm.Canonical } func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil.RequestResponseMap, dgst string) { @@ -525,6 +519,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil Headers: http.Header(map[string][]string{ "Content-Length": {"0"}, "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + "Content-Type": {schema1.MediaTypeManifest}, }), } } else { @@ -534,6 +529,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil Headers: http.Header(map[string][]string{ "Content-Length": {fmt.Sprint(len(content))}, "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + "Content-Type": {schema1.MediaTypeManifest}, }), } @@ -553,6 +549,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request Headers: http.Header(map[string][]string{ "Content-Length": {fmt.Sprint(len(content))}, "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + "Content-Type": {schema1.MediaTypeManifest}, }), }, }) @@ -566,6 +563,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request Headers: http.Header(map[string][]string{ "Content-Length": {fmt.Sprint(len(content))}, "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + "Content-Type": {schema1.MediaTypeManifest}, }), }, }) @@ -598,12 +596,17 @@ func checkEqualManifest(m1, m2 *schema1.SignedManifest) error { return nil } -func TestManifestFetch(t *testing.T) { +func TestV1ManifestFetch(t *testing.T) { ctx := context.Background() repo := "test.example.com/repo" m1, dgst, _ := newRandomSchemaV1Manifest(repo, "latest", 6) var m testutil.RequestResponseMap - addTestManifest(repo, dgst.String(), m1.Raw, &m) + _, pl, err := m1.Payload() + if err != nil { + t.Fatal(err) + } + addTestManifest(repo, dgst.String(), pl, &m) + addTestManifest(repo, "latest", pl, &m) e, c := testServer(m) defer c() @@ -617,7 +620,7 @@ func TestManifestFetch(t *testing.T) { t.Fatal(err) } - ok, err := ms.Exists(dgst) + ok, err := ms.Exists(ctx, dgst) if err != nil { t.Fatal(err) } @@ -625,11 +628,29 @@ func TestManifestFetch(t *testing.T) { t.Fatal("Manifest does not exist") } - manifest, err := ms.Get(dgst) + manifest, err := ms.Get(ctx, dgst) if err != nil { t.Fatal(err) } - if err := checkEqualManifest(manifest, m1); err != nil { + v1manifest, ok := manifest.(*schema1.SignedManifest) + if !ok { + t.Fatalf("Unexpected manifest type from Get: %T", manifest) + } + + if err := checkEqualManifest(v1manifest, m1); err != nil { + t.Fatal(err) + } + + manifest, err = ms.Get(ctx, dgst, WithTag("latest")) + if err != nil { + t.Fatal(err) + } + v1manifest, ok = manifest.(*schema1.SignedManifest) + if !ok { + t.Fatalf("Unexpected manifest type from Get: %T", manifest) + } + + if err = checkEqualManifest(v1manifest, m1); err != nil { t.Fatal(err) } } @@ -643,17 +664,22 @@ func TestManifestFetchWithEtag(t *testing.T) { e, c := testServer(m) defer c() - r, err := NewRepository(context.Background(), repo, e, nil) + ctx := context.Background() + r, err := NewRepository(ctx, repo, e, nil) if err != nil { t.Fatal(err) } - ctx := context.Background() + ms, err := r.Manifests(ctx) if err != nil { t.Fatal(err) } - _, err = ms.GetByTag("latest", AddEtagToTag("latest", d1.String())) + clientManifestService, ok := ms.(*manifests) + if !ok { + panic("wrong type for client manifest service") + } + _, err = clientManifestService.Get(ctx, d1, WithTag("latest"), AddEtagToTag("latest", d1.String())) if err != distribution.ErrManifestNotModified { t.Fatal(err) } @@ -690,10 +716,10 @@ func TestManifestDelete(t *testing.T) { t.Fatal(err) } - if err := ms.Delete(dgst1); err != nil { + if err := ms.Delete(ctx, dgst1); err != nil { t.Fatal(err) } - if err := ms.Delete(dgst2); err == nil { + if err := ms.Delete(ctx, dgst2); err == nil { t.Fatal("Expected error deleting unknown manifest") } // TODO(dmcgowan): Check for specific unknown error @@ -702,12 +728,17 @@ func TestManifestDelete(t *testing.T) { func TestManifestPut(t *testing.T) { repo := "test.example.com/repo/delete" m1, dgst, _ := newRandomSchemaV1Manifest(repo, "other", 6) + + _, payload, err := m1.Payload() + if err != nil { + t.Fatal(err) + } var m testutil.RequestResponseMap m = append(m, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "PUT", Route: "/v2/" + repo + "/manifests/other", - Body: m1.Raw, + Body: payload, }, Response: testutil.Response{ StatusCode: http.StatusAccepted, @@ -731,7 +762,7 @@ func TestManifestPut(t *testing.T) { t.Fatal(err) } - if err := ms.Put(m1); err != nil { + if _, err := ms.Put(ctx, m1, WithTag(m1.Tag)); err != nil { t.Fatal(err) } @@ -751,21 +782,22 @@ func TestManifestTags(t *testing.T) { } `)) var m testutil.RequestResponseMap - m = append(m, testutil.RequestResponseMapping{ - Request: testutil.Request{ - Method: "GET", - Route: "/v2/" + repo + "/tags/list", - }, - Response: testutil.Response{ - StatusCode: http.StatusOK, - Body: tagsList, - Headers: http.Header(map[string][]string{ - "Content-Length": {fmt.Sprint(len(tagsList))}, - "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, - }), - }, - }) - + for i := 0; i < 3; i++ { + m = append(m, testutil.RequestResponseMapping{ + Request: testutil.Request{ + Method: "GET", + Route: "/v2/" + repo + "/tags/list", + }, + Response: testutil.Response{ + StatusCode: http.StatusOK, + Body: tagsList, + Headers: http.Header(map[string][]string{ + "Content-Length": {fmt.Sprint(len(tagsList))}, + "Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)}, + }), + }, + }) + } e, c := testServer(m) defer c() @@ -773,22 +805,29 @@ func TestManifestTags(t *testing.T) { if err != nil { t.Fatal(err) } + ctx := context.Background() - ms, err := r.Manifests(ctx) + tagService := r.Tags(ctx) + + tags, err := tagService.All(ctx) if err != nil { t.Fatal(err) } - - tags, err := ms.Tags() - if err != nil { - t.Fatal(err) - } - if len(tags) != 3 { t.Fatalf("Wrong number of tags returned: %d, expected 3", len(tags)) } - // TODO(dmcgowan): Check array + expected := map[string]struct{}{ + "tag1": {}, + "tag2": {}, + "funtag": {}, + } + for _, t := range tags { + delete(expected, t) + } + if len(expected) != 0 { + t.Fatalf("unexpected tags returned: %v", expected) + } // TODO(dmcgowan): Check for error cases } @@ -821,7 +860,7 @@ func TestManifestUnauthorized(t *testing.T) { t.Fatal(err) } - _, err = ms.Get(dgst) + _, err = ms.Get(ctx, dgst) if err == nil { t.Fatal("Expected error fetching manifest") } diff --git a/docs/handlers/api_test.go b/docs/handlers/api_test.go index 7b7c3c0d..2672b77b 100644 --- a/docs/handlers/api_test.go +++ b/docs/handlers/api_test.go @@ -871,19 +871,15 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m t.Fatalf("unexpected error signing manifest: %v", err) } - payload, err := signedManifest.Payload() - checkErr(t, err, "getting manifest payload") - - dgst := digest.FromBytes(payload) - + dgst := digest.FromBytes(signedManifest.Canonical) args.signedManifest = signedManifest args.dgst = dgst manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) checkErr(t, err, "building manifest url") - resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest) - checkResponse(t, "putting signed manifest", resp, http.StatusCreated) + resp = putManifest(t, "putting signed manifest no error", manifestURL, signedManifest) + checkResponse(t, "putting signed manifest no error", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, @@ -914,11 +910,12 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m var fetchedManifest schema1.SignedManifest dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&fetchedManifest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } - if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) { + if !bytes.Equal(fetchedManifest.Canonical, signedManifest.Canonical) { t.Fatalf("manifests do not match") } @@ -940,10 +937,55 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m t.Fatalf("error decoding fetched manifest: %v", err) } - if !bytes.Equal(fetchedManifestByDigest.Raw, signedManifest.Raw) { + if !bytes.Equal(fetchedManifestByDigest.Canonical, signedManifest.Canonical) { t.Fatalf("manifests do not match") } + // check signature was roundtripped + signatures, err := fetchedManifestByDigest.Signatures() + if err != nil { + t.Fatal(err) + } + + if len(signatures) != 1 { + t.Fatalf("expected 1 signature from manifest, got: %d", len(signatures)) + } + + // Re-sign, push and pull the same digest + sm2, err := schema1.Sign(&fetchedManifestByDigest.Manifest, env.pk) + if err != nil { + t.Fatal(err) + + } + + resp = putManifest(t, "re-putting signed manifest", manifestDigestURL, sm2) + checkResponse(t, "re-putting signed manifest", resp, http.StatusCreated) + + resp, err = http.Get(manifestDigestURL) + checkErr(t, err, "re-fetching manifest by digest") + defer resp.Body.Close() + + checkResponse(t, "re-fetching uploaded manifest", resp, http.StatusOK) + checkHeaders(t, resp, http.Header{ + "Docker-Content-Digest": []string{dgst.String()}, + "ETag": []string{fmt.Sprintf(`"%s"`, dgst)}, + }) + + dec = json.NewDecoder(resp.Body) + if err := dec.Decode(&fetchedManifestByDigest); err != nil { + t.Fatalf("error decoding fetched manifest: %v", err) + } + + // check two signatures were roundtripped + signatures, err = fetchedManifestByDigest.Signatures() + if err != nil { + t.Fatal(err) + } + + if len(signatures) != 2 { + t.Fatalf("expected 2 signature from manifest, got: %d", len(signatures)) + } + // Get by name with etag, gives 304 etag := resp.Header.Get("Etag") req, err := http.NewRequest("GET", manifestURL, nil) @@ -956,7 +998,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m t.Fatalf("Error constructing request: %s", err) } - checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified) + checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified) // Get by digest with etag, gives 304 req, err = http.NewRequest("GET", manifestDigestURL, nil) @@ -969,7 +1011,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m t.Fatalf("Error constructing request: %s", err) } - checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified) + checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified) // Ensure that the tag is listed. resp, err = http.Get(tagsURL) @@ -1143,8 +1185,13 @@ func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *te func putManifest(t *testing.T, msg, url string, v interface{}) *http.Response { var body []byte + if sm, ok := v.(*schema1.SignedManifest); ok { - body = sm.Raw + _, pl, err := sm.Payload() + if err != nil { + t.Fatalf("error getting payload: %v", err) + } + body = pl } else { var err error body, err = json.MarshalIndent(v, "", " ") @@ -1435,7 +1482,7 @@ func checkErr(t *testing.T, err error, msg string) { } } -func createRepository(env *testEnv, t *testing.T, imageName string, tag string) { +func createRepository(env *testEnv, t *testing.T, imageName string, tag string) digest.Digest { unsignedManifest := &schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, @@ -1459,7 +1506,6 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string) for i := range unsignedManifest.FSLayers { rs, dgstStr, err := testutil.CreateRandomTarFile() - if err != nil { t.Fatalf("error creating random layer %d: %v", i, err) } @@ -1477,20 +1523,22 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string) t.Fatalf("unexpected error signing manifest: %v", err) } - payload, err := signedManifest.Payload() - checkErr(t, err, "getting manifest payload") + dgst := digest.FromBytes(signedManifest.Canonical) - dgst := digest.FromBytes(payload) - - manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) + // Create this repository by tag to ensure the tag mapping is made in the registry + manifestDigestURL, err := env.builder.BuildManifestURL(imageName, tag) checkErr(t, err, "building manifest url") + location, err := env.builder.BuildManifestURL(imageName, dgst.String()) + checkErr(t, err, "building location URL") + resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ - "Location": []string{manifestDigestURL}, + "Location": []string{location}, "Docker-Content-Digest": []string{dgst.String()}, }) + return dgst } // Test mutation operations on a registry configured as a cache. Ensure that they return @@ -1577,3 +1625,64 @@ func TestCheckContextNotifier(t *testing.T) { t.Fatalf("wrong status code - expected 200, got %d", resp.StatusCode) } } + +func TestProxyManifestGetByTag(t *testing.T) { + truthConfig := configuration.Configuration{ + Storage: configuration.Storage{ + "inmemory": configuration.Parameters{}, + }, + } + truthConfig.HTTP.Headers = headerConfig + + imageName := "foo/bar" + tag := "latest" + + truthEnv := newTestEnvWithConfig(t, &truthConfig) + // create a repository in the truth registry + dgst := createRepository(truthEnv, t, imageName, tag) + + proxyConfig := configuration.Configuration{ + Storage: configuration.Storage{ + "inmemory": configuration.Parameters{}, + }, + Proxy: configuration.Proxy{ + RemoteURL: truthEnv.server.URL, + }, + } + proxyConfig.HTTP.Headers = headerConfig + + proxyEnv := newTestEnvWithConfig(t, &proxyConfig) + + manifestDigestURL, err := proxyEnv.builder.BuildManifestURL(imageName, dgst.String()) + checkErr(t, err, "building manifest url") + + resp, err := http.Get(manifestDigestURL) + checkErr(t, err, "fetching manifest from proxy by digest") + defer resp.Body.Close() + + manifestTagURL, err := proxyEnv.builder.BuildManifestURL(imageName, tag) + checkErr(t, err, "building manifest url") + + resp, err = http.Get(manifestTagURL) + checkErr(t, err, "fetching manifest from proxy by tag") + defer resp.Body.Close() + checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK) + checkHeaders(t, resp, http.Header{ + "Docker-Content-Digest": []string{dgst.String()}, + }) + + // Create another manifest in the remote with the same image/tag pair + newDigest := createRepository(truthEnv, t, imageName, tag) + if dgst == newDigest { + t.Fatalf("non-random test data") + } + + // fetch it with the same proxy URL as before. Ensure the updated content is at the same tag + resp, err = http.Get(manifestTagURL) + checkErr(t, err, "fetching manifest from proxy by tag") + defer resp.Body.Close() + checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK) + checkHeaders(t, resp, http.Header{ + "Docker-Content-Digest": []string{newDigest.String()}, + }) +} diff --git a/docs/handlers/images.go b/docs/handlers/images.go index 2ec51b99..be14b00a 100644 --- a/docs/handlers/images.go +++ b/docs/handlers/images.go @@ -2,19 +2,15 @@ package handlers import ( "bytes" - "encoding/json" "fmt" "net/http" - "strings" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/registry/api/errcode" "github.com/docker/distribution/registry/api/v2" "github.com/gorilla/handlers" - "golang.org/x/net/context" ) // imageManifestDispatcher takes the request context and builds the @@ -33,7 +29,8 @@ func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler { } mhandler := handlers.MethodHandler{ - "GET": http.HandlerFunc(imageManifestHandler.GetImageManifest), + "GET": http.HandlerFunc(imageManifestHandler.GetImageManifest), + "HEAD": http.HandlerFunc(imageManifestHandler.GetImageManifest), } if !ctx.readOnly { @@ -54,6 +51,8 @@ type imageManifestHandler struct { } // GetImageManifest fetches the image manifest from the storage backend, if it exists. +// todo(richardscothern): this assumes v2 schema 1 manifests for now but in the future +// get the version from the Accept HTTP header func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(imh).Debug("GetImageManifest") manifests, err := imh.Repository.Manifests(imh) @@ -62,42 +61,38 @@ func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http return } - var sm *schema1.SignedManifest + var manifest distribution.Manifest if imh.Tag != "" { - sm, err = manifests.GetByTag(imh.Tag) - } else { - if etagMatch(r, imh.Digest.String()) { - w.WriteHeader(http.StatusNotModified) + tags := imh.Repository.Tags(imh) + desc, err := tags.Get(imh, imh.Tag) + if err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) return } - sm, err = manifests.Get(imh.Digest) + imh.Digest = desc.Digest } + if etagMatch(r, imh.Digest.String()) { + w.WriteHeader(http.StatusNotModified) + return + } + + manifest, err = manifests.Get(imh, imh.Digest) if err != nil { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) return } - // Get the digest, if we don't already have it. - if imh.Digest == "" { - dgst, err := digestManifest(imh, sm) - if err != nil { - imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err)) - return - } - if etagMatch(r, dgst.String()) { - w.WriteHeader(http.StatusNotModified) - return - } - - imh.Digest = dgst + ct, p, err := manifest.Payload() + if err != nil { + return } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Header().Set("Content-Length", fmt.Sprint(len(sm.Raw))) + w.Header().Set("Content-Type", ct) + w.Header().Set("Content-Length", fmt.Sprint(len(p))) w.Header().Set("Docker-Content-Digest", imh.Digest.String()) w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest)) - w.Write(sm.Raw) + w.Write(p) } func etagMatch(r *http.Request, etag string) bool { @@ -109,7 +104,7 @@ func etagMatch(r *http.Request, etag string) bool { return false } -// PutImageManifest validates and stores and image in the registry. +// PutImageManifest validates and stores an image in the registry. func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(imh).Debug("PutImageManifest") manifests, err := imh.Repository.Manifests(imh) @@ -124,39 +119,28 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http return } - var manifest schema1.SignedManifest - if err := json.Unmarshal(jsonBuf.Bytes(), &manifest); err != nil { + mediaType := r.Header.Get("Content-Type") + manifest, desc, err := distribution.UnmarshalManifest(mediaType, jsonBuf.Bytes()) + if err != nil { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err)) return } - dgst, err := digestManifest(imh, &manifest) - if err != nil { - imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err)) - return - } - - // Validate manifest tag or digest matches payload - if imh.Tag != "" { - if manifest.Tag != imh.Tag { - ctxu.GetLogger(imh).Errorf("invalid tag on manifest payload: %q != %q", manifest.Tag, imh.Tag) - imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid) - return - } - - imh.Digest = dgst - } else if imh.Digest != "" { - if dgst != imh.Digest { - ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", dgst, imh.Digest) + if imh.Digest != "" { + if desc.Digest != imh.Digest { + ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", desc.Digest, imh.Digest) imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid) return } + } else if imh.Tag != "" { + imh.Digest = desc.Digest } else { imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail("no tag or digest specified")) return } - if err := manifests.Put(&manifest); err != nil { + _, err = manifests.Put(imh, manifest) + if err != nil { // TODO(stevvooe): These error handling switches really need to be // handled by an app global mapper. if err == distribution.ErrUnsupported { @@ -188,6 +172,17 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http return } + // Tag this manifest + if imh.Tag != "" { + tags := imh.Repository.Tags(imh) + err = tags.Tag(imh, imh.Tag, desc) + if err != nil { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + return + } + + } + // Construct a canonical url for the uploaded manifest. location, err := imh.urlBuilder.BuildManifestURL(imh.Repository.Name(), imh.Digest.String()) if err != nil { @@ -212,7 +207,7 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h return } - err = manifests.Delete(imh.Digest) + err = manifests.Delete(imh, imh.Digest) if err != nil { switch err { case digest.ErrDigestUnsupported: @@ -233,22 +228,3 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h w.WriteHeader(http.StatusAccepted) } - -// digestManifest takes a digest of the given manifest. This belongs somewhere -// better but we'll wait for a refactoring cycle to find that real somewhere. -func digestManifest(ctx context.Context, sm *schema1.SignedManifest) (digest.Digest, error) { - p, err := sm.Payload() - if err != nil { - if !strings.Contains(err.Error(), "missing signature key") { - ctxu.GetLogger(ctx).Errorf("error getting manifest payload: %v", err) - return "", err - } - - // NOTE(stevvooe): There are no signatures but we still have a - // payload. The request will fail later but this is not the - // responsibility of this part of the code. - p = sm.Raw - } - - return digest.FromBytes(p), nil -} diff --git a/docs/handlers/tags.go b/docs/handlers/tags.go index 54725585..d9f0106c 100644 --- a/docs/handlers/tags.go +++ b/docs/handlers/tags.go @@ -34,13 +34,9 @@ type tagsAPIResponse struct { // GetTags returns a json list of tags for a specific image name. func (th *tagsHandler) GetTags(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() - manifests, err := th.Repository.Manifests(th) - if err != nil { - th.Errors = append(th.Errors, err) - return - } - tags, err := manifests.Tags() + tagService := th.Repository.Tags(th) + tags, err := tagService.All(th) if err != nil { switch err := err.(type) { case distribution.ErrRepositoryUnknown: diff --git a/docs/proxy/proxymanifeststore.go b/docs/proxy/proxymanifeststore.go index 1e9e24de..13cb5f6b 100644 --- a/docs/proxy/proxymanifeststore.go +++ b/docs/proxy/proxymanifeststore.go @@ -6,8 +6,6 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" - "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/proxy/scheduler" ) @@ -24,8 +22,8 @@ type proxyManifestStore struct { var _ distribution.ManifestService = &proxyManifestStore{} -func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) { - exists, err := pms.localManifests.Exists(dgst) +func (pms proxyManifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { + exists, err := pms.localManifests.Exists(ctx, dgst) if err != nil { return false, err } @@ -33,117 +31,56 @@ func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) { return true, nil } - return pms.remoteManifests.Exists(dgst) + return pms.remoteManifests.Exists(ctx, dgst) } -func (pms proxyManifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { - sm, err := pms.localManifests.Get(dgst) - if err == nil { - proxyMetrics.ManifestPush(uint64(len(sm.Raw))) - return sm, err +func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { + // At this point `dgst` was either specified explicitly, or returned by the + // tagstore with the most recent association. + var fromRemote bool + manifest, err := pms.localManifests.Get(ctx, dgst, options...) + if err != nil { + manifest, err = pms.remoteManifests.Get(ctx, dgst, options...) + if err != nil { + return nil, err + } + fromRemote = true } - sm, err = pms.remoteManifests.Get(dgst) + _, payload, err := manifest.Payload() if err != nil { return nil, err } - proxyMetrics.ManifestPull(uint64(len(sm.Raw))) - err = pms.localManifests.Put(sm) - if err != nil { - return nil, err + proxyMetrics.ManifestPush(uint64(len(payload))) + if fromRemote { + proxyMetrics.ManifestPull(uint64(len(payload))) + + _, err = pms.localManifests.Put(ctx, manifest) + if err != nil { + return nil, err + } + + // Schedule the repo for removal + pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL) + + // Ensure the manifest blob is cleaned up + pms.scheduler.AddBlob(dgst.String(), repositoryTTL) } - // Schedule the repo for removal - pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL) - - // Ensure the manifest blob is cleaned up - pms.scheduler.AddBlob(dgst.String(), repositoryTTL) - - proxyMetrics.ManifestPush(uint64(len(sm.Raw))) - - return sm, err + return manifest, err } -func (pms proxyManifestStore) Tags() ([]string, error) { - return pms.localManifests.Tags() +func (pms proxyManifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { + var d digest.Digest + return d, distribution.ErrUnsupported } -func (pms proxyManifestStore) ExistsByTag(tag string) (bool, error) { - exists, err := pms.localManifests.ExistsByTag(tag) - if err != nil { - return false, err - } - if exists { - return true, nil - } - - return pms.remoteManifests.ExistsByTag(tag) -} - -func (pms proxyManifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { - var localDigest digest.Digest - - localManifest, err := pms.localManifests.GetByTag(tag, options...) - switch err.(type) { - case distribution.ErrManifestUnknown, distribution.ErrManifestUnknownRevision: - goto fromremote - case nil: - break - default: - return nil, err - } - - localDigest, err = manifestDigest(localManifest) - if err != nil { - return nil, err - } - -fromremote: - var sm *schema1.SignedManifest - sm, err = pms.remoteManifests.GetByTag(tag, client.AddEtagToTag(tag, localDigest.String())) - if err != nil && err != distribution.ErrManifestNotModified { - return nil, err - } - - if err == distribution.ErrManifestNotModified { - context.GetLogger(pms.ctx).Debugf("Local manifest for %q is latest, dgst=%s", tag, localDigest.String()) - return localManifest, nil - } - context.GetLogger(pms.ctx).Debugf("Updated manifest for %q, dgst=%s", tag, localDigest.String()) - - err = pms.localManifests.Put(sm) - if err != nil { - return nil, err - } - - dgst, err := manifestDigest(sm) - if err != nil { - return nil, err - } - pms.scheduler.AddBlob(dgst.String(), repositoryTTL) - pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL) - - proxyMetrics.ManifestPull(uint64(len(sm.Raw))) - proxyMetrics.ManifestPush(uint64(len(sm.Raw))) - - return sm, err -} - -func manifestDigest(sm *schema1.SignedManifest) (digest.Digest, error) { - payload, err := sm.Payload() - if err != nil { - return "", err - - } - - return digest.FromBytes(payload), nil -} - -func (pms proxyManifestStore) Put(manifest *schema1.SignedManifest) error { +func (pms proxyManifestStore) Delete(ctx context.Context, dgst digest.Digest) error { return distribution.ErrUnsupported } -func (pms proxyManifestStore) Delete(dgst digest.Digest) error { - return distribution.ErrUnsupported +/*func (pms proxyManifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + return 0, distribution.ErrUnsupported } +*/ diff --git a/docs/proxy/proxymanifeststore_test.go b/docs/proxy/proxymanifeststore_test.go index a5a0a21b..aeecae10 100644 --- a/docs/proxy/proxymanifeststore_test.go +++ b/docs/proxy/proxymanifeststore_test.go @@ -37,40 +37,31 @@ func (te manifestStoreTestEnv) RemoteStats() *map[string]int { return &rs } -func (sm statsManifest) Delete(dgst digest.Digest) error { +func (sm statsManifest) Delete(ctx context.Context, dgst digest.Digest) error { sm.stats["delete"]++ - return sm.manifests.Delete(dgst) + return sm.manifests.Delete(ctx, dgst) } -func (sm statsManifest) Exists(dgst digest.Digest) (bool, error) { +func (sm statsManifest) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { sm.stats["exists"]++ - return sm.manifests.Exists(dgst) + return sm.manifests.Exists(ctx, dgst) } -func (sm statsManifest) ExistsByTag(tag string) (bool, error) { - sm.stats["existbytag"]++ - return sm.manifests.ExistsByTag(tag) -} - -func (sm statsManifest) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { +func (sm statsManifest) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { sm.stats["get"]++ - return sm.manifests.Get(dgst) + return sm.manifests.Get(ctx, dgst) } -func (sm statsManifest) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { - sm.stats["getbytag"]++ - return sm.manifests.GetByTag(tag, options...) -} - -func (sm statsManifest) Put(manifest *schema1.SignedManifest) error { +func (sm statsManifest) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { sm.stats["put"]++ - return sm.manifests.Put(manifest) + return sm.manifests.Put(ctx, manifest) } -func (sm statsManifest) Tags() ([]string, error) { - sm.stats["tags"]++ - return sm.manifests.Tags() +/*func (sm statsManifest) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + sm.stats["enumerate"]++ + return sm.manifests.Enumerate(ctx, manifests, last) } +*/ func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv { ctx := context.Background() @@ -169,15 +160,12 @@ func populateRepo(t *testing.T, ctx context.Context, repository distribution.Rep if err != nil { t.Fatalf(err.Error()) } - ms.Put(sm) + dgst, err := ms.Put(ctx, sm) if err != nil { t.Fatalf("unexpected errors putting manifest: %v", err) } - pl, err := sm.Payload() - if err != nil { - t.Fatal(err) - } - return digest.FromBytes(pl), nil + + return dgst, nil } // TestProxyManifests contains basic acceptance tests @@ -189,8 +177,9 @@ func TestProxyManifests(t *testing.T) { localStats := env.LocalStats() remoteStats := env.RemoteStats() + ctx := context.Background() // Stat - must check local and remote - exists, err := env.manifests.ExistsByTag("latest") + exists, err := env.manifests.Exists(ctx, env.manifestDigest) if err != nil { t.Fatalf("Error checking existance") } @@ -198,15 +187,16 @@ func TestProxyManifests(t *testing.T) { t.Errorf("Unexpected non-existant manifest") } - if (*localStats)["existbytag"] != 1 && (*remoteStats)["existbytag"] != 1 { - t.Errorf("Unexpected exists count") + if (*localStats)["exists"] != 1 && (*remoteStats)["exists"] != 1 { + t.Errorf("Unexpected exists count : \n%v \n%v", localStats, remoteStats) } // Get - should succeed and pull manifest into local - _, err = env.manifests.Get(env.manifestDigest) + _, err = env.manifests.Get(ctx, env.manifestDigest) if err != nil { t.Fatal(err) } + if (*localStats)["get"] != 1 && (*remoteStats)["get"] != 1 { t.Errorf("Unexpected get count") } @@ -216,7 +206,7 @@ func TestProxyManifests(t *testing.T) { } // Stat - should only go to local - exists, err = env.manifests.ExistsByTag("latest") + exists, err = env.manifests.Exists(ctx, env.manifestDigest) if err != nil { t.Fatal(err) } @@ -224,19 +214,21 @@ func TestProxyManifests(t *testing.T) { t.Errorf("Unexpected non-existant manifest") } - if (*localStats)["existbytag"] != 2 && (*remoteStats)["existbytag"] != 1 { + if (*localStats)["exists"] != 2 && (*remoteStats)["exists"] != 1 { t.Errorf("Unexpected exists count") - } // Get - should get from remote, to test freshness - _, err = env.manifests.Get(env.manifestDigest) + _, err = env.manifests.Get(ctx, env.manifestDigest) if err != nil { t.Fatal(err) } - if (*remoteStats)["get"] != 2 && (*remoteStats)["existsbytag"] != 1 && (*localStats)["put"] != 1 { + if (*remoteStats)["get"] != 2 && (*remoteStats)["exists"] != 1 && (*localStats)["put"] != 1 { t.Errorf("Unexpected get count") } +} + +func TestProxyTagService(t *testing.T) { } diff --git a/docs/proxy/proxyregistry.go b/docs/proxy/proxyregistry.go index 8a5f5ef6..8e1be5f2 100644 --- a/docs/proxy/proxyregistry.go +++ b/docs/proxy/proxyregistry.go @@ -42,6 +42,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name s.OnManifestExpire(func(repoName string) error { return v.RemoveRepository(repoName) }) + err = s.Start() if err != nil { return nil, err @@ -78,7 +79,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri if err != nil { return nil, err } - localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification) + localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification()) if err != nil { return nil, err } @@ -106,8 +107,11 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri ctx: ctx, scheduler: pr.scheduler, }, - name: name, - signatures: localRepo.Signatures(), + name: name, + tags: proxyTagService{ + localTags: localRepo.Tags(ctx), + remoteTags: remoteRepo.Tags(ctx), + }, }, nil } @@ -115,14 +119,13 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri // locally, or pulling it through from a remote and caching it locally if it doesn't // already exist type proxiedRepository struct { - blobStore distribution.BlobStore - manifests distribution.ManifestService - name string - signatures distribution.SignatureService + blobStore distribution.BlobStore + manifests distribution.ManifestService + name string + tags distribution.TagService } func (pr *proxiedRepository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) { - // options return pr.manifests, nil } @@ -134,6 +137,6 @@ func (pr *proxiedRepository) Name() string { return pr.name } -func (pr *proxiedRepository) Signatures() distribution.SignatureService { - return pr.signatures +func (pr *proxiedRepository) Tags(ctx context.Context) distribution.TagService { + return pr.tags } diff --git a/docs/proxy/proxytagservice.go b/docs/proxy/proxytagservice.go new file mode 100644 index 00000000..c52460c4 --- /dev/null +++ b/docs/proxy/proxytagservice.go @@ -0,0 +1,58 @@ +package proxy + +import ( + "github.com/docker/distribution" + "github.com/docker/distribution/context" +) + +// proxyTagService supports local and remote lookup of tags. +type proxyTagService struct { + localTags distribution.TagService + remoteTags distribution.TagService +} + +var _ distribution.TagService = proxyTagService{} + +// Get attempts to get the most recent digest for the tag by checking the remote +// tag service first and then caching it locally. If the remote is unavailable +// the local association is returned +func (pt proxyTagService) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + desc, err := pt.remoteTags.Get(ctx, tag) + if err == nil { + err := pt.localTags.Tag(ctx, tag, desc) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil + } + + desc, err = pt.localTags.Get(ctx, tag) + if err != nil { + return distribution.Descriptor{}, err + } + return desc, nil +} + +func (pt proxyTagService) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + return distribution.ErrUnsupported +} + +func (pt proxyTagService) Untag(ctx context.Context, tag string) error { + err := pt.localTags.Untag(ctx, tag) + if err != nil { + return err + } + return nil +} + +func (pt proxyTagService) All(ctx context.Context) ([]string, error) { + tags, err := pt.remoteTags.All(ctx) + if err == nil { + return tags, err + } + return pt.localTags.All(ctx) +} + +func (pt proxyTagService) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + return []string{}, distribution.ErrUnsupported +} diff --git a/docs/proxy/proxytagservice_test.go b/docs/proxy/proxytagservice_test.go new file mode 100644 index 00000000..8d9518c0 --- /dev/null +++ b/docs/proxy/proxytagservice_test.go @@ -0,0 +1,164 @@ +package proxy + +import ( + "sort" + "sync" + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" +) + +type mockTagStore struct { + mapping map[string]distribution.Descriptor + sync.Mutex +} + +var _ distribution.TagService = &mockTagStore{} + +func (m *mockTagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { + m.Lock() + defer m.Unlock() + + if d, ok := m.mapping[tag]; ok { + return d, nil + } + return distribution.Descriptor{}, distribution.ErrTagUnknown{} +} + +func (m *mockTagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { + m.Lock() + defer m.Unlock() + + m.mapping[tag] = desc + return nil +} + +func (m *mockTagStore) Untag(ctx context.Context, tag string) error { + m.Lock() + defer m.Unlock() + + if _, ok := m.mapping[tag]; ok { + delete(m.mapping, tag) + return nil + } + return distribution.ErrTagUnknown{} +} + +func (m *mockTagStore) All(ctx context.Context) ([]string, error) { + m.Lock() + defer m.Unlock() + + var tags []string + for tag := range m.mapping { + tags = append(tags, tag) + } + + return tags, nil +} + +func (m *mockTagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + panic("not implemented") +} + +func testProxyTagService(local, remote map[string]distribution.Descriptor) *proxyTagService { + if local == nil { + local = make(map[string]distribution.Descriptor) + } + if remote == nil { + remote = make(map[string]distribution.Descriptor) + } + return &proxyTagService{ + localTags: &mockTagStore{mapping: local}, + remoteTags: &mockTagStore{mapping: remote}, + } +} + +func TestGet(t *testing.T) { + remoteDesc := distribution.Descriptor{Size: 42} + remoteTag := "remote" + proxyTags := testProxyTagService(map[string]distribution.Descriptor{remoteTag: remoteDesc}, nil) + + ctx := context.Background() + + // Get pre-loaded tag + d, err := proxyTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal(err) + } + + if d != remoteDesc { + t.Fatal("unable to get put tag") + } + + local, err := proxyTags.localTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal("remote tag not pulled into store") + } + + if local != remoteDesc { + t.Fatalf("unexpected descriptor pulled through") + } + + // Manually overwrite remote tag + newRemoteDesc := distribution.Descriptor{Size: 43} + err = proxyTags.remoteTags.Tag(ctx, remoteTag, newRemoteDesc) + if err != nil { + t.Fatal(err) + } + + d, err = proxyTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal(err) + } + + if d != newRemoteDesc { + t.Fatal("unable to get put tag") + } + + _, err = proxyTags.localTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal("remote tag not pulled into store") + } + + // untag, ensure it's removed locally, but present in remote + err = proxyTags.Untag(ctx, remoteTag) + if err != nil { + t.Fatal(err) + } + + _, err = proxyTags.localTags.Get(ctx, remoteTag) + if err == nil { + t.Fatalf("Expected error getting Untag'd tag") + } + + _, err = proxyTags.remoteTags.Get(ctx, remoteTag) + if err != nil { + t.Fatalf("remote tag should not be untagged with proxyTag.Untag") + } + + _, err = proxyTags.Get(ctx, remoteTag) + if err != nil { + t.Fatal("untagged tag should be pulled through") + } + + // Add another tag. Ensure both tags appear in enumerate + err = proxyTags.remoteTags.Tag(ctx, "funtag", distribution.Descriptor{Size: 42}) + if err != nil { + t.Fatal(err) + } + + all, err := proxyTags.All(ctx) + if err != nil { + t.Fatal(err) + } + + if len(all) != 2 { + t.Fatalf("Unexpected tag length returned from All() : %d ", len(all)) + } + + sort.Strings(all) + if all[0] != "funtag" && all[1] != "remote" { + t.Fatalf("Unexpected tags returned from All() : %v ", all) + } +} diff --git a/docs/storage/manifeststore.go b/docs/storage/manifeststore.go index 024c8e4b..73061592 100644 --- a/docs/storage/manifeststore.go +++ b/docs/storage/manifeststore.go @@ -1,6 +1,7 @@ package storage import ( + "encoding/json" "fmt" "github.com/docker/distribution" @@ -11,20 +12,21 @@ import ( "github.com/docker/libtrust" ) +// manifestStore is a storage driver based store for storing schema1 manifests. type manifestStore struct { repository *repository - revisionStore *revisionStore - tagStore *tagStore + blobStore *linkedBlobStore ctx context.Context + signatures *signatureStore skipDependencyVerification bool } var _ distribution.ManifestService = &manifestStore{} -func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) { +func (ms *manifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) { context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists") - _, err := ms.revisionStore.blobStore.Stat(ms.ctx, dgst) + _, err := ms.blobStore.Stat(ms.ctx, dgst) if err != nil { if err == distribution.ErrBlobUnknown { return false, nil @@ -36,76 +38,131 @@ func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) { return true, nil } -func (ms *manifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) { +func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) { context.GetLogger(ms.ctx).Debug("(*manifestStore).Get") - return ms.revisionStore.get(ms.ctx, dgst) + // Ensure that this revision is available in this repository. + _, err := ms.blobStore.Stat(ctx, dgst) + if err != nil { + if err == distribution.ErrBlobUnknown { + return nil, distribution.ErrManifestUnknownRevision{ + Name: ms.repository.Name(), + Revision: dgst, + } + } + + return nil, err + } + + // TODO(stevvooe): Need to check descriptor from above to ensure that the + // mediatype is as we expect for the manifest store. + + content, err := ms.blobStore.Get(ctx, dgst) + if err != nil { + if err == distribution.ErrBlobUnknown { + return nil, distribution.ErrManifestUnknownRevision{ + Name: ms.repository.Name(), + Revision: dgst, + } + } + + return nil, err + } + + // Fetch the signatures for the manifest + signatures, err := ms.signatures.Get(dgst) + if err != nil { + return nil, err + } + + jsig, err := libtrust.NewJSONSignature(content, signatures...) + if err != nil { + return nil, err + } + + // Extract the pretty JWS + raw, err := jsig.PrettySignature("signatures") + if err != nil { + return nil, err + } + + var sm schema1.SignedManifest + if err := json.Unmarshal(raw, &sm); err != nil { + return nil, err + } + + return &sm, nil } -// SkipLayerVerification allows a manifest to be Put before it's +// SkipLayerVerification allows a manifest to be Put before its // layers are on the filesystem -func SkipLayerVerification(ms distribution.ManifestService) error { - if ms, ok := ms.(*manifestStore); ok { +func SkipLayerVerification() distribution.ManifestServiceOption { + return skipLayerOption{} +} + +type skipLayerOption struct{} + +func (o skipLayerOption) Apply(m distribution.ManifestService) error { + if ms, ok := m.(*manifestStore); ok { ms.skipDependencyVerification = true return nil } return fmt.Errorf("skip layer verification only valid for manifestStore") } -func (ms *manifestStore) Put(manifest *schema1.SignedManifest) error { +func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) { context.GetLogger(ms.ctx).Debug("(*manifestStore).Put") - if err := ms.verifyManifest(ms.ctx, manifest); err != nil { - return err + sm, ok := manifest.(*schema1.SignedManifest) + if !ok { + return "", fmt.Errorf("non-v1 manifest put to signed manifestStore: %T", manifest) } - // Store the revision of the manifest - revision, err := ms.revisionStore.put(ms.ctx, manifest) + if err := ms.verifyManifest(ms.ctx, *sm); err != nil { + return "", err + } + + mt := schema1.MediaTypeManifest + payload := sm.Canonical + + revision, err := ms.blobStore.Put(ctx, mt, payload) if err != nil { - return err + context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + return "", err } - // Now, tag the manifest - return ms.tagStore.tag(manifest.Tag, revision.Digest) + // Link the revision into the repository. + if err := ms.blobStore.linkBlob(ctx, revision); err != nil { + return "", err + } + + // Grab each json signature and store them. + signatures, err := sm.Signatures() + if err != nil { + return "", err + } + + if err := ms.signatures.Put(revision.Digest, signatures...); err != nil { + return "", err + } + + return revision.Digest, nil } // Delete removes the revision of the specified manfiest. -func (ms *manifestStore) Delete(dgst digest.Digest) error { +func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error { context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete") - return ms.revisionStore.delete(ms.ctx, dgst) + return ms.blobStore.Delete(ctx, dgst) } -func (ms *manifestStore) Tags() ([]string, error) { - context.GetLogger(ms.ctx).Debug("(*manifestStore).Tags") - return ms.tagStore.tags() -} - -func (ms *manifestStore) ExistsByTag(tag string) (bool, error) { - context.GetLogger(ms.ctx).Debug("(*manifestStore).ExistsByTag") - return ms.tagStore.exists(tag) -} - -func (ms *manifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) { - for _, option := range options { - err := option(ms) - if err != nil { - return nil, err - } - } - - context.GetLogger(ms.ctx).Debug("(*manifestStore).GetByTag") - dgst, err := ms.tagStore.resolve(tag) - if err != nil { - return nil, err - } - - return ms.revisionStore.get(ms.ctx, dgst) +func (ms *manifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) { + return 0, distribution.ErrUnsupported } // verifyManifest ensures that the manifest content is valid from the // perspective of the registry. It ensures that the signature is valid for the // enclosed payload. As a policy, the registry only tries to store valid -// content, leaving trust policies of that content up to consumers. -func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.SignedManifest) error { +// content, leaving trust policies of that content up to consumems. +func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest) error { var errs distribution.ErrManifestVerification if len(mnfst.Name) > reference.NameTotalLengthMax { @@ -129,7 +186,7 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign len(mnfst.History), len(mnfst.FSLayers))) } - if _, err := schema1.Verify(mnfst); err != nil { + if _, err := schema1.Verify(&mnfst); err != nil { switch err { case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey: errs = append(errs, distribution.ErrManifestUnverified{}) @@ -143,15 +200,15 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign } if !ms.skipDependencyVerification { - for _, fsLayer := range mnfst.FSLayers { - _, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.BlobSum) + for _, fsLayer := range mnfst.References() { + _, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest) if err != nil { if err != distribution.ErrBlobUnknown { errs = append(errs, err) } - // On error here, we always append unknown blob errors. - errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.BlobSum}) + // On error here, we always append unknown blob erroms. + errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest}) } } } diff --git a/docs/storage/manifeststore_test.go b/docs/storage/manifeststore_test.go index de31b364..a41feb04 100644 --- a/docs/storage/manifeststore_test.go +++ b/docs/storage/manifeststore_test.go @@ -30,7 +30,8 @@ type manifestStoreTestEnv struct { func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv { ctx := context.Background() driver := inmemory.New() - registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect) + registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider( + memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } @@ -58,24 +59,6 @@ func TestManifestStorage(t *testing.T) { t.Fatal(err) } - exists, err := ms.ExistsByTag(env.tag) - if err != nil { - t.Fatalf("unexpected error checking manifest existence: %v", err) - } - - if exists { - t.Fatalf("manifest should not exist") - } - - if _, err := ms.GetByTag(env.tag); true { - switch err.(type) { - case distribution.ErrManifestUnknown: - break - default: - t.Fatalf("expected manifest unknown error: %#v", err) - } - } - m := schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, @@ -114,7 +97,7 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("error signing manifest: %v", err) } - err = ms.Put(sm) + _, err = ms.Put(ctx, sm) if err == nil { t.Fatalf("expected errors putting manifest with full verification") } @@ -150,30 +133,40 @@ func TestManifestStorage(t *testing.T) { } } - if err = ms.Put(sm); err != nil { + var manifestDigest digest.Digest + if manifestDigest, err = ms.Put(ctx, sm); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } - exists, err = ms.ExistsByTag(env.tag) + exists, err := ms.Exists(ctx, manifestDigest) if err != nil { - t.Fatalf("unexpected error checking manifest existence: %v", err) + t.Fatalf("unexpected error checking manifest existence: %#v", err) } if !exists { t.Fatalf("manifest should exist") } - fetchedManifest, err := ms.GetByTag(env.tag) - + fromStore, err := ms.Get(ctx, manifestDigest) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } + fetchedManifest, ok := fromStore.(*schema1.SignedManifest) + if !ok { + t.Fatalf("unexpected manifest type from signedstore") + } + if !reflect.DeepEqual(fetchedManifest, sm) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm) } - fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures") + _, pl, err := fetchedManifest.Payload() + if err != nil { + t.Fatalf("error getting payload %#v", err) + } + + fetchedJWS, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } @@ -185,8 +178,9 @@ func TestManifestStorage(t *testing.T) { // Now that we have a payload, take a moment to check that the manifest is // return by the payload digest. + dgst := digest.FromBytes(payload) - exists, err = ms.Exists(dgst) + exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("error checking manifest existence by digest: %v", err) } @@ -195,7 +189,7 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("manifest %s should exist", dgst) } - fetchedByDigest, err := ms.Get(dgst) + fetchedByDigest, err := ms.Get(ctx, dgst) if err != nil { t.Fatalf("unexpected error fetching manifest by digest: %v", err) } @@ -213,20 +207,6 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1) } - // Grabs the tags and check that this tagged manifest is present - tags, err := ms.Tags() - if err != nil { - t.Fatalf("unexpected error fetching tags: %v", err) - } - - if len(tags) != 1 { - t.Fatalf("unexpected tags returned: %v", tags) - } - - if tags[0] != env.tag { - t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag}) - } - // Now, push the same manifest with a different key pk2, err := libtrust.GenerateECP256PrivateKey() if err != nil { @@ -237,8 +217,12 @@ func TestManifestStorage(t *testing.T) { if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } + _, pl, err = sm2.Payload() + if err != nil { + t.Fatalf("error getting payload %#v", err) + } - jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures") + jws2, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("error parsing signature: %v", err) } @@ -252,15 +236,20 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1) } - if err = ms.Put(sm2); err != nil { + if manifestDigest, err = ms.Put(ctx, sm2); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } - fetched, err := ms.GetByTag(env.tag) + fromStore, err = ms.Get(ctx, manifestDigest) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } + fetched, ok := fromStore.(*schema1.SignedManifest) + if !ok { + t.Fatalf("unexpected type from signed manifeststore : %T", fetched) + } + if _, err := schema1.Verify(fetched); err != nil { t.Fatalf("unexpected error verifying manifest: %v", err) } @@ -276,7 +265,12 @@ func TestManifestStorage(t *testing.T) { t.Fatalf("unexpected error getting expected signatures: %v", err) } - receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures") + _, pl, err = fetched.Payload() + if err != nil { + t.Fatalf("error getting payload %#v", err) + } + + receivedJWS, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } @@ -302,12 +296,12 @@ func TestManifestStorage(t *testing.T) { } // Test deleting manifests - err = ms.Delete(dgst) + err = ms.Delete(ctx, dgst) if err != nil { t.Fatalf("unexpected an error deleting manifest by digest: %v", err) } - exists, err = ms.Exists(dgst) + exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("Error querying manifest existence") } @@ -315,7 +309,7 @@ func TestManifestStorage(t *testing.T) { t.Errorf("Deleted manifest should not exist") } - deletedManifest, err := ms.Get(dgst) + deletedManifest, err := ms.Get(ctx, dgst) if err == nil { t.Errorf("Unexpected success getting deleted manifest") } @@ -331,12 +325,12 @@ func TestManifestStorage(t *testing.T) { } // Re-upload should restore manifest to a good state - err = ms.Put(sm) + _, err = ms.Put(ctx, sm) if err != nil { t.Errorf("Error re-uploading deleted manifest") } - exists, err = ms.Exists(dgst) + exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("Error querying manifest existence") } @@ -344,7 +338,7 @@ func TestManifestStorage(t *testing.T) { t.Errorf("Restored manifest should exist") } - deletedManifest, err = ms.Get(dgst) + deletedManifest, err = ms.Get(ctx, dgst) if err != nil { t.Errorf("Unexpected error getting manifest") } @@ -364,7 +358,7 @@ func TestManifestStorage(t *testing.T) { if err != nil { t.Fatal(err) } - err = ms.Delete(dgst) + err = ms.Delete(ctx, dgst) if err == nil { t.Errorf("Unexpected success deleting while disabled") } diff --git a/docs/storage/registry.go b/docs/storage/registry.go index 5ef06d53..c58b91d8 100644 --- a/docs/storage/registry.go +++ b/docs/storage/registry.go @@ -145,6 +145,15 @@ func (repo *repository) Name() string { return repo.name } +func (repo *repository) Tags(ctx context.Context) distribution.TagService { + tags := &tagStore{ + repository: repo, + blobStore: repo.registry.blobStore, + } + + return tags +} + // Manifests returns an instance of ManifestService. Instantiation is cheap and // may be context sensitive in the future. The instance should be used similar // to a request local. @@ -159,36 +168,31 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M ms := &manifestStore{ ctx: ctx, repository: repo, - revisionStore: &revisionStore{ - ctx: ctx, - repository: repo, - blobStore: &linkedBlobStore{ - ctx: ctx, - blobStore: repo.blobStore, - repository: repo, - deleteEnabled: repo.registry.deleteEnabled, - blobAccessController: &linkedBlobStatter{ - blobStore: repo.blobStore, - repository: repo, - linkPathFns: manifestLinkPathFns, - }, - - // TODO(stevvooe): linkPath limits this blob store to only - // manifests. This instance cannot be used for blob checks. - linkPathFns: manifestLinkPathFns, - resumableDigestEnabled: repo.resumableDigestEnabled, + blobStore: &linkedBlobStore{ + ctx: ctx, + blobStore: repo.blobStore, + repository: repo, + deleteEnabled: repo.registry.deleteEnabled, + blobAccessController: &linkedBlobStatter{ + blobStore: repo.blobStore, + repository: repo, + linkPathFns: manifestLinkPathFns, }, + + // TODO(stevvooe): linkPath limits this blob store to only + // manifests. This instance cannot be used for blob checks. + linkPathFns: manifestLinkPathFns, }, - tagStore: &tagStore{ + signatures: &signatureStore{ ctx: ctx, repository: repo, - blobStore: repo.registry.blobStore, + blobStore: repo.blobStore, }, } // Apply options for _, option := range options { - err := option(ms) + err := option.Apply(ms) if err != nil { return nil, err } @@ -225,11 +229,3 @@ func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore { resumableDigestEnabled: repo.resumableDigestEnabled, } } - -func (repo *repository) Signatures() distribution.SignatureService { - return &signatureStore{ - repository: repo, - blobStore: repo.blobStore, - ctx: repo.ctx, - } -} diff --git a/docs/storage/revisionstore.go b/docs/storage/revisionstore.go deleted file mode 100644 index ed2d5dd3..00000000 --- a/docs/storage/revisionstore.go +++ /dev/null @@ -1,111 +0,0 @@ -package storage - -import ( - "encoding/json" - - "github.com/docker/distribution" - "github.com/docker/distribution/context" - "github.com/docker/distribution/digest" - "github.com/docker/distribution/manifest/schema1" - "github.com/docker/libtrust" -) - -// revisionStore supports storing and managing manifest revisions. -type revisionStore struct { - repository *repository - blobStore *linkedBlobStore - ctx context.Context -} - -// get retrieves the manifest, keyed by revision digest. -func (rs *revisionStore) get(ctx context.Context, revision digest.Digest) (*schema1.SignedManifest, error) { - // Ensure that this revision is available in this repository. - _, err := rs.blobStore.Stat(ctx, revision) - if err != nil { - if err == distribution.ErrBlobUnknown { - return nil, distribution.ErrManifestUnknownRevision{ - Name: rs.repository.Name(), - Revision: revision, - } - } - - return nil, err - } - - // TODO(stevvooe): Need to check descriptor from above to ensure that the - // mediatype is as we expect for the manifest store. - - content, err := rs.blobStore.Get(ctx, revision) - if err != nil { - if err == distribution.ErrBlobUnknown { - return nil, distribution.ErrManifestUnknownRevision{ - Name: rs.repository.Name(), - Revision: revision, - } - } - - return nil, err - } - - // Fetch the signatures for the manifest - signatures, err := rs.repository.Signatures().Get(revision) - if err != nil { - return nil, err - } - - jsig, err := libtrust.NewJSONSignature(content, signatures...) - if err != nil { - return nil, err - } - - // Extract the pretty JWS - raw, err := jsig.PrettySignature("signatures") - if err != nil { - return nil, err - } - - var sm schema1.SignedManifest - if err := json.Unmarshal(raw, &sm); err != nil { - return nil, err - } - - return &sm, nil -} - -// put stores the manifest in the repository, if not already present. Any -// updated signatures will be stored, as well. -func (rs *revisionStore) put(ctx context.Context, sm *schema1.SignedManifest) (distribution.Descriptor, error) { - // Resolve the payload in the manifest. - payload, err := sm.Payload() - if err != nil { - return distribution.Descriptor{}, err - } - - // Digest and store the manifest payload in the blob store. - revision, err := rs.blobStore.Put(ctx, schema1.ManifestMediaType, payload) - if err != nil { - context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) - return distribution.Descriptor{}, err - } - - // Link the revision into the repository. - if err := rs.blobStore.linkBlob(ctx, revision); err != nil { - return distribution.Descriptor{}, err - } - - // Grab each json signature and store them. - signatures, err := sm.Signatures() - if err != nil { - return distribution.Descriptor{}, err - } - - if err := rs.repository.Signatures().Put(revision.Digest, signatures...); err != nil { - return distribution.Descriptor{}, err - } - - return revision, nil -} - -func (rs *revisionStore) delete(ctx context.Context, revision digest.Digest) error { - return rs.blobStore.Delete(ctx, revision) -} diff --git a/docs/storage/signaturestore.go b/docs/storage/signaturestore.go index f5888f64..ede4e0e2 100644 --- a/docs/storage/signaturestore.go +++ b/docs/storage/signaturestore.go @@ -4,7 +4,6 @@ import ( "path" "sync" - "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/docker/distribution/digest" ) @@ -15,16 +14,6 @@ type signatureStore struct { ctx context.Context } -func newSignatureStore(ctx context.Context, repo *repository, blobStore *blobStore) *signatureStore { - return &signatureStore{ - ctx: ctx, - repository: repo, - blobStore: blobStore, - } -} - -var _ distribution.SignatureService = &signatureStore{} - func (s *signatureStore) Get(dgst digest.Digest) ([][]byte, error) { signaturesPath, err := pathFor(manifestSignaturesPathSpec{ name: s.repository.Name(), diff --git a/docs/storage/tagstore.go b/docs/storage/tagstore.go index aec95286..167c7fa0 100644 --- a/docs/storage/tagstore.go +++ b/docs/storage/tagstore.go @@ -9,37 +9,41 @@ import ( storagedriver "github.com/docker/distribution/registry/storage/driver" ) +var _ distribution.TagService = &tagStore{} + // tagStore provides methods to manage manifest tags in a backend storage driver. +// This implementation uses the same on-disk layout as the (now deleted) tag +// store. This provides backward compatibility with current registry deployments +// which only makes use of the Digest field of the returned distribution.Descriptor +// but does not enable full roundtripping of Descriptor objects type tagStore struct { repository *repository blobStore *blobStore - ctx context.Context } -// tags lists the manifest tags for the specified repository. -func (ts *tagStore) tags() ([]string, error) { - p, err := pathFor(manifestTagPathSpec{ +// All returns all tags +func (ts *tagStore) All(ctx context.Context) ([]string, error) { + var tags []string + + pathSpec, err := pathFor(manifestTagPathSpec{ name: ts.repository.Name(), }) - if err != nil { - return nil, err + return tags, err } - var tags []string - entries, err := ts.blobStore.driver.List(ts.ctx, p) + entries, err := ts.blobStore.driver.List(ctx, pathSpec) if err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: - return nil, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()} + return tags, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()} default: - return nil, err + return tags, err } } for _, entry := range entries { _, filename := path.Split(entry) - tags = append(tags, filename) } @@ -47,7 +51,7 @@ func (ts *tagStore) tags() ([]string, error) { } // exists returns true if the specified manifest tag exists in the repository. -func (ts *tagStore) exists(tag string) (bool, error) { +func (ts *tagStore) exists(ctx context.Context, tag string) (bool, error) { tagPath, err := pathFor(manifestTagCurrentPathSpec{ name: ts.repository.Name(), tag: tag, @@ -57,7 +61,7 @@ func (ts *tagStore) exists(tag string) (bool, error) { return false, err } - exists, err := exists(ts.ctx, ts.blobStore.driver, tagPath) + exists, err := exists(ctx, ts.blobStore.driver, tagPath) if err != nil { return false, err } @@ -65,9 +69,9 @@ func (ts *tagStore) exists(tag string) (bool, error) { return exists, nil } -// tag tags the digest with the given tag, updating the the store to point at +// Tag tags the digest with the given tag, updating the the store to point at // the current tag. The digest must point to a manifest. -func (ts *tagStore) tag(tag string, revision digest.Digest) error { +func (ts *tagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error { currentPath, err := pathFor(manifestTagCurrentPathSpec{ name: ts.repository.Name(), tag: tag, @@ -77,43 +81,44 @@ func (ts *tagStore) tag(tag string, revision digest.Digest) error { return err } - nbs := ts.linkedBlobStore(ts.ctx, tag) + lbs := ts.linkedBlobStore(ctx, tag) + // Link into the index - if err := nbs.linkBlob(ts.ctx, distribution.Descriptor{Digest: revision}); err != nil { + if err := lbs.linkBlob(ctx, desc); err != nil { return err } // Overwrite the current link - return ts.blobStore.link(ts.ctx, currentPath, revision) + return ts.blobStore.link(ctx, currentPath, desc.Digest) } // resolve the current revision for name and tag. -func (ts *tagStore) resolve(tag string) (digest.Digest, error) { +func (ts *tagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) { currentPath, err := pathFor(manifestTagCurrentPathSpec{ name: ts.repository.Name(), tag: tag, }) if err != nil { - return "", err + return distribution.Descriptor{}, err } - revision, err := ts.blobStore.readlink(ts.ctx, currentPath) + revision, err := ts.blobStore.readlink(ctx, currentPath) if err != nil { switch err.(type) { case storagedriver.PathNotFoundError: - return "", distribution.ErrManifestUnknown{Name: ts.repository.Name(), Tag: tag} + return distribution.Descriptor{}, distribution.ErrTagUnknown{Tag: tag} } - return "", err + return distribution.Descriptor{}, err } - return revision, nil + return distribution.Descriptor{Digest: revision}, nil } // delete removes the tag from repository, including the history of all // revisions that have the specified tag. -func (ts *tagStore) delete(tag string) error { +func (ts *tagStore) Untag(ctx context.Context, tag string) error { tagPath, err := pathFor(manifestTagPathSpec{ name: ts.repository.Name(), tag: tag, @@ -123,7 +128,7 @@ func (ts *tagStore) delete(tag string) error { return err } - return ts.blobStore.driver.Delete(ts.ctx, tagPath) + return ts.blobStore.driver.Delete(ctx, tagPath) } // linkedBlobStore returns the linkedBlobStore for the named tag, allowing one @@ -145,3 +150,10 @@ func (ts *tagStore) linkedBlobStore(ctx context.Context, tag string) *linkedBlob }}, } } + +// Lookup recovers a list of tags which refer to this digest. When a manifest is deleted by +// digest, tag entries which point to it need to be recovered to avoid dangling tags. +func (ts *tagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) { + // An efficient implementation of this will require changes to the S3 driver. + return make([]string, 0), nil +} diff --git a/docs/storage/tagstore_test.go b/docs/storage/tagstore_test.go new file mode 100644 index 00000000..79660199 --- /dev/null +++ b/docs/storage/tagstore_test.go @@ -0,0 +1,150 @@ +package storage + +import ( + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/storage/driver/inmemory" +) + +type tagsTestEnv struct { + ts distribution.TagService + ctx context.Context +} + +func testTagStore(t *testing.T) *tagsTestEnv { + ctx := context.Background() + d := inmemory.New() + reg, err := NewRegistry(ctx, d) + if err != nil { + t.Fatal(err) + } + + repo, err := reg.Repository(ctx, "a/b") + if err != nil { + t.Fatal(err) + } + + return &tagsTestEnv{ + ctx: ctx, + ts: repo.Tags(ctx), + } +} + +func TestTagStoreTag(t *testing.T) { + env := testTagStore(t) + tags := env.ts + ctx := env.ctx + + d := distribution.Descriptor{} + err := tags.Tag(ctx, "latest", d) + if err == nil { + t.Errorf("unexpected error putting malformed descriptor : %s", err) + } + + d.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + err = tags.Tag(ctx, "latest", d) + if err != nil { + t.Error(err) + } + + d1, err := tags.Get(ctx, "latest") + if err != nil { + t.Error(err) + } + + if d1.Digest != d.Digest { + t.Error("put and get digest differ") + } + + // Overwrite existing + d.Digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + err = tags.Tag(ctx, "latest", d) + if err != nil { + t.Error(err) + } + + d1, err = tags.Get(ctx, "latest") + if err != nil { + t.Error(err) + } + + if d1.Digest != d.Digest { + t.Error("put and get digest differ") + } +} + +func TestTagStoreUnTag(t *testing.T) { + env := testTagStore(t) + tags := env.ts + ctx := env.ctx + desc := distribution.Descriptor{Digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"} + + err := tags.Untag(ctx, "latest") + if err == nil { + t.Errorf("Expected error untagging non-existant tag") + } + + err = tags.Tag(ctx, "latest", desc) + if err != nil { + t.Error(err) + } + + err = tags.Untag(ctx, "latest") + if err != nil { + t.Error(err) + } + + _, err = tags.Get(ctx, "latest") + if err == nil { + t.Error("Expected error getting untagged tag") + } +} + +func TestTagAll(t *testing.T) { + env := testTagStore(t) + tagStore := env.ts + ctx := env.ctx + + alpha := "abcdefghijklmnopqrstuvwxyz" + for i := 0; i < len(alpha); i++ { + tag := alpha[i] + desc := distribution.Descriptor{Digest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"} + err := tagStore.Tag(ctx, string(tag), desc) + if err != nil { + t.Error(err) + } + } + + all, err := tagStore.All(ctx) + if err != nil { + t.Error(err) + } + if len(all) != len(alpha) { + t.Errorf("Unexpected count returned from enumerate") + } + + for i, c := range all { + if c != string(alpha[i]) { + t.Errorf("unexpected tag in enumerate %s", c) + } + } + + removed := "a" + err = tagStore.Untag(ctx, removed) + if err != nil { + t.Error(err) + } + + all, err = tagStore.All(ctx) + if err != nil { + t.Error(err) + } + for _, tag := range all { + if tag == removed { + t.Errorf("unexpected tag in enumerate %s", removed) + } + } + +}