From 7ef71988a8e3c4fb51041ac813c00b46bb706016 Mon Sep 17 00:00:00 2001
From: Aaron Lehmann <aaron.lehmann@docker.com>
Date: Wed, 16 Dec 2015 17:26:13 -0800
Subject: [PATCH] Add support for manifest list ("fat manifest")
Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>
---
docs/handlers/api_test.go | 193 ++++++++++++++++++++++++-
docs/storage/manifestlisthandler.go | 96 ++++++++++++
docs/storage/manifeststore.go | 24 ++-
docs/storage/registry.go | 5 +
docs/storage/schema2manifesthandler.go | 5 +-
docs/storage/signedmanifesthandler.go | 2 +-
6 files changed, 313 insertions(+), 12 deletions(-)
create mode 100644 docs/storage/manifestlisthandler.go
diff --git a/docs/handlers/api_test.go b/docs/handlers/api_test.go
index e38b4da8..0393c8f1 100644
--- a/docs/handlers/api_test.go
+++ b/docs/handlers/api_test.go
@@ -23,6 +23,7 @@ import (
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
+ "github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/registry/api/errcode"
@@ -702,12 +703,14 @@ func TestManifestAPI(t *testing.T) {
deleteEnabled := false
env := newTestEnv(t, deleteEnabled)
testManifestAPISchema1(t, env, "foo/schema1")
- testManifestAPISchema2(t, env, "foo/schema2")
+ schema2Args := testManifestAPISchema2(t, env, "foo/schema2")
+ testManifestAPIManifestList(t, env, schema2Args)
deleteEnabled = true
env = newTestEnv(t, deleteEnabled)
testManifestAPISchema1(t, env, "foo/schema1")
- testManifestAPISchema2(t, env, "foo/schema2")
+ schema2Args = testManifestAPISchema2(t, env, "foo/schema2")
+ testManifestAPIManifestList(t, env, schema2Args)
}
func TestManifestDelete(t *testing.T) {
@@ -1393,6 +1396,179 @@ func testManifestAPISchema2(t *testing.T, env *testEnv, imageName string) manife
return args
}
+func testManifestAPIManifestList(t *testing.T, env *testEnv, args manifestArgs) {
+ imageName := args.imageName
+ tag := "manifestlisttag"
+
+ manifestURL, err := env.builder.BuildManifestURL(imageName, tag)
+ if err != nil {
+ t.Fatalf("unexpected error getting manifest url: %v", err)
+ }
+
+ // --------------------------------
+ // Attempt to push manifest list that refers to an unknown manifest
+ manifestList := &manifestlist.ManifestList{
+ Versioned: manifest.Versioned{
+ SchemaVersion: 2,
+ },
+ MediaType: manifestlist.MediaTypeManifestList,
+ Manifests: []manifestlist.ManifestDescriptor{
+ {
+ Descriptor: distribution.Descriptor{
+ Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
+ Size: 3253,
+ MediaType: schema2.MediaTypeManifest,
+ },
+ Platform: manifestlist.PlatformSpec{
+ Architecture: "amd64",
+ OS: "linux",
+ },
+ },
+ },
+ }
+
+ resp := putManifest(t, "putting missing manifest manifestlist", manifestURL, manifestlist.MediaTypeManifestList, manifestList)
+ defer resp.Body.Close()
+ checkResponse(t, "putting missing manifest manifestlist", resp, http.StatusBadRequest)
+ _, p, counts := checkBodyHasErrorCodes(t, "putting missing manifest manifestlist", resp, v2.ErrorCodeManifestBlobUnknown)
+
+ expectedCounts := map[errcode.ErrorCode]int{
+ v2.ErrorCodeManifestBlobUnknown: 1,
+ }
+
+ if !reflect.DeepEqual(counts, expectedCounts) {
+ t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p))
+ }
+
+ // -------------------
+ // Push a manifest list that references an actual manifest
+ manifestList.Manifests[0].Digest = args.dgst
+ deserializedManifestList, err := manifestlist.FromDescriptors(manifestList.Manifests)
+ if err != nil {
+ t.Fatalf("could not create DeserializedManifestList: %v", err)
+ }
+ _, canonical, err := deserializedManifestList.Payload()
+ if err != nil {
+ t.Fatalf("could not get manifest list payload: %v", err)
+ }
+ dgst := digest.FromBytes(canonical)
+
+ manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
+ checkErr(t, err, "building manifest url")
+
+ resp = putManifest(t, "putting manifest list no error", manifestURL, manifestlist.MediaTypeManifestList, deserializedManifestList)
+ checkResponse(t, "putting manifest list no error", resp, http.StatusCreated)
+ checkHeaders(t, resp, http.Header{
+ "Location": []string{manifestDigestURL},
+ "Docker-Content-Digest": []string{dgst.String()},
+ })
+
+ // --------------------
+ // Push by digest -- should get same result
+ resp = putManifest(t, "putting manifest list by digest", manifestDigestURL, manifestlist.MediaTypeManifestList, deserializedManifestList)
+ checkResponse(t, "putting manifest list by digest", resp, http.StatusCreated)
+ checkHeaders(t, resp, http.Header{
+ "Location": []string{manifestDigestURL},
+ "Docker-Content-Digest": []string{dgst.String()},
+ })
+
+ // ------------------
+ // Fetch by tag name
+ req, err := http.NewRequest("GET", manifestURL, nil)
+ if err != nil {
+ t.Fatalf("Error constructing request: %s", err)
+ }
+ req.Header.Set("Accept", manifestlist.MediaTypeManifestList)
+ req.Header.Add("Accept", schema1.MediaTypeManifest)
+ req.Header.Add("Accept", schema2.MediaTypeManifest)
+ resp, err = http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("unexpected error fetching manifest list: %v", err)
+ }
+ defer resp.Body.Close()
+
+ checkResponse(t, "fetching uploaded manifest list", resp, http.StatusOK)
+ checkHeaders(t, resp, http.Header{
+ "Docker-Content-Digest": []string{dgst.String()},
+ "ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
+ })
+
+ var fetchedManifestList manifestlist.DeserializedManifestList
+ dec := json.NewDecoder(resp.Body)
+
+ if err := dec.Decode(&fetchedManifestList); err != nil {
+ t.Fatalf("error decoding fetched manifest list: %v", err)
+ }
+
+ _, fetchedCanonical, err := fetchedManifestList.Payload()
+ if err != nil {
+ t.Fatalf("error getting manifest list payload: %v", err)
+ }
+
+ if !bytes.Equal(fetchedCanonical, canonical) {
+ t.Fatalf("manifest lists do not match")
+ }
+
+ // ---------------
+ // Fetch by digest
+ req, err = http.NewRequest("GET", manifestDigestURL, nil)
+ if err != nil {
+ t.Fatalf("Error constructing request: %s", err)
+ }
+ req.Header.Set("Accept", manifestlist.MediaTypeManifestList)
+ resp, err = http.DefaultClient.Do(req)
+ checkErr(t, err, "fetching manifest list by digest")
+ defer resp.Body.Close()
+
+ checkResponse(t, "fetching uploaded manifest list", resp, http.StatusOK)
+ checkHeaders(t, resp, http.Header{
+ "Docker-Content-Digest": []string{dgst.String()},
+ "ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
+ })
+
+ var fetchedManifestListByDigest manifestlist.DeserializedManifestList
+ dec = json.NewDecoder(resp.Body)
+ if err := dec.Decode(&fetchedManifestListByDigest); err != nil {
+ t.Fatalf("error decoding fetched manifest: %v", err)
+ }
+
+ _, fetchedCanonical, err = fetchedManifestListByDigest.Payload()
+ if err != nil {
+ t.Fatalf("error getting manifest list payload: %v", err)
+ }
+
+ if !bytes.Equal(fetchedCanonical, canonical) {
+ t.Fatalf("manifests do not match")
+ }
+
+ // Get by name with etag, gives 304
+ etag := resp.Header.Get("Etag")
+ req, err = http.NewRequest("GET", manifestURL, nil)
+ if err != nil {
+ t.Fatalf("Error constructing request: %s", err)
+ }
+ req.Header.Set("If-None-Match", etag)
+ resp, err = http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("Error constructing request: %s", err)
+ }
+
+ checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified)
+
+ // Get by digest with etag, gives 304
+ req, err = http.NewRequest("GET", manifestDigestURL, nil)
+ if err != nil {
+ t.Fatalf("Error constructing request: %s", err)
+ }
+ req.Header.Set("If-None-Match", etag)
+ resp, err = http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("Error constructing request: %s", err)
+ }
+
+ checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified)
+}
+
func testManifestDelete(t *testing.T, env *testEnv, args manifestArgs) {
imageName := args.imageName
dgst := args.dgst
@@ -1521,13 +1697,20 @@ func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *te
func putManifest(t *testing.T, msg, url, contentType string, v interface{}) *http.Response {
var body []byte
- if sm, ok := v.(*schema1.SignedManifest); ok {
- _, pl, err := sm.Payload()
+ switch m := v.(type) {
+ case *schema1.SignedManifest:
+ _, pl, err := m.Payload()
if err != nil {
t.Fatalf("error getting payload: %v", err)
}
body = pl
- } else {
+ case *manifestlist.DeserializedManifestList:
+ _, pl, err := m.Payload()
+ if err != nil {
+ t.Fatalf("error getting payload: %v", err)
+ }
+ body = pl
+ default:
var err error
body, err = json.MarshalIndent(v, "", " ")
if err != nil {
diff --git a/docs/storage/manifestlisthandler.go b/docs/storage/manifestlisthandler.go
new file mode 100644
index 00000000..42027d13
--- /dev/null
+++ b/docs/storage/manifestlisthandler.go
@@ -0,0 +1,96 @@
+package storage
+
+import (
+ "fmt"
+
+ "encoding/json"
+ "github.com/docker/distribution"
+ "github.com/docker/distribution/context"
+ "github.com/docker/distribution/digest"
+ "github.com/docker/distribution/manifest/manifestlist"
+)
+
+// manifestListHandler is a ManifestHandler that covers schema2 manifest lists.
+type manifestListHandler struct {
+ repository *repository
+ blobStore *linkedBlobStore
+ ctx context.Context
+}
+
+var _ ManifestHandler = &manifestListHandler{}
+
+func (ms *manifestListHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
+ context.GetLogger(ms.ctx).Debug("(*manifestListHandler).Unmarshal")
+
+ var m manifestlist.DeserializedManifestList
+ if err := json.Unmarshal(content, &m); err != nil {
+ return nil, err
+ }
+
+ return &m, nil
+}
+
+func (ms *manifestListHandler) Put(ctx context.Context, manifestList distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) {
+ context.GetLogger(ms.ctx).Debug("(*manifestListHandler).Put")
+
+ m, ok := manifestList.(*manifestlist.DeserializedManifestList)
+ if !ok {
+ return "", fmt.Errorf("wrong type put to manifestListHandler: %T", manifestList)
+ }
+
+ if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil {
+ return "", err
+ }
+
+ mt, payload, err := m.Payload()
+ if err != nil {
+ return "", err
+ }
+
+ revision, err := ms.blobStore.Put(ctx, mt, payload)
+ if err != nil {
+ context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
+ return "", err
+ }
+
+ // Link the revision into the repository.
+ if err := ms.blobStore.linkBlob(ctx, revision); err != nil {
+ return "", err
+ }
+
+ return revision.Digest, nil
+}
+
+// verifyManifest ensures that the manifest content is valid from the
+// perspective of the registry. As a policy, the registry only tries to
+// store valid content, leaving trust policies of that content up to
+// consumers.
+func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst manifestlist.DeserializedManifestList, skipDependencyVerification bool) error {
+ var errs distribution.ErrManifestVerification
+
+ if !skipDependencyVerification {
+ // This manifest service is different from the blob service
+ // returned by Blob. It uses a linked blob store to ensure that
+ // only manifests are accessible.
+ manifestService, err := ms.repository.Manifests(ctx)
+ if err != nil {
+ return err
+ }
+
+ for _, manifestDescriptor := range mnfst.References() {
+ exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
+ if err != nil && err != distribution.ErrBlobUnknown {
+ errs = append(errs, err)
+ }
+ if err != nil || !exists {
+ // On error here, we always append unknown blob errors.
+ errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest})
+ }
+ }
+ }
+ if len(errs) != 0 {
+ return errs
+ }
+
+ return nil
+}
diff --git a/docs/storage/manifeststore.go b/docs/storage/manifeststore.go
index cd3aa43e..cd01670b 100644
--- a/docs/storage/manifeststore.go
+++ b/docs/storage/manifeststore.go
@@ -8,6 +8,7 @@ import (
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest"
+ "github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
)
@@ -44,8 +45,9 @@ type manifestStore struct {
skipDependencyVerification bool
- schema1Handler ManifestHandler
- schema2Handler ManifestHandler
+ schema1Handler ManifestHandler
+ schema2Handler ManifestHandler
+ manifestListHandler ManifestHandler
}
var _ distribution.ManifestService = &manifestStore{}
@@ -92,7 +94,21 @@ func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ..
case 1:
return ms.schema1Handler.Unmarshal(ctx, dgst, content)
case 2:
- return ms.schema2Handler.Unmarshal(ctx, dgst, content)
+ // This can be an image manifest or a manifest list
+ var mediaType struct {
+ MediaType string `json:"mediaType"`
+ }
+ if err = json.Unmarshal(content, &mediaType); err != nil {
+ return nil, err
+ }
+ switch mediaType.MediaType {
+ case schema2.MediaTypeManifest:
+ return ms.schema2Handler.Unmarshal(ctx, dgst, content)
+ case manifestlist.MediaTypeManifestList:
+ return ms.manifestListHandler.Unmarshal(ctx, dgst, content)
+ default:
+ return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", mediaType.MediaType)}
+ }
}
return nil, fmt.Errorf("unrecognized manifest schema version %d", versioned.SchemaVersion)
@@ -106,6 +122,8 @@ func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest
return ms.schema1Handler.Put(ctx, manifest, ms.skipDependencyVerification)
case *schema2.DeserializedManifest:
return ms.schema2Handler.Put(ctx, manifest, ms.skipDependencyVerification)
+ case *manifestlist.DeserializedManifestList:
+ return ms.manifestListHandler.Put(ctx, manifest, ms.skipDependencyVerification)
}
return "", fmt.Errorf("unrecognized manifest type %T", manifest)
diff --git a/docs/storage/registry.go b/docs/storage/registry.go
index d22c6c81..b3810676 100644
--- a/docs/storage/registry.go
+++ b/docs/storage/registry.go
@@ -200,6 +200,11 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
repository: repo,
blobStore: blobStore,
},
+ manifestListHandler: &manifestListHandler{
+ ctx: ctx,
+ repository: repo,
+ blobStore: blobStore,
+ },
}
// Apply options
diff --git a/docs/storage/schema2manifesthandler.go b/docs/storage/schema2manifesthandler.go
index 9cec2e81..115786e2 100644
--- a/docs/storage/schema2manifesthandler.go
+++ b/docs/storage/schema2manifesthandler.go
@@ -62,9 +62,8 @@ func (ms *schema2ManifestHandler) Put(ctx context.Context, manifest distribution
}
// verifyManifest ensures that the manifest content is valid from the
-// perspective of the registry. It ensures that the signature is valid for the
-// enclosed payload. As a policy, the registry only tries to store valid
-// content, leaving trust policies of that content up to consumems.
+// perspective of the registry. As a policy, the registry only tries to store
+// valid content, leaving trust policies of that content up to consumers.
func (ms *schema2ManifestHandler) verifyManifest(ctx context.Context, mnfst schema2.DeserializedManifest, skipDependencyVerification bool) error {
var errs distribution.ErrManifestVerification
diff --git a/docs/storage/signedmanifesthandler.go b/docs/storage/signedmanifesthandler.go
index a375516a..02663226 100644
--- a/docs/storage/signedmanifesthandler.go
+++ b/docs/storage/signedmanifesthandler.go
@@ -91,7 +91,7 @@ func (ms *signedManifestHandler) Put(ctx context.Context, manifest distribution.
// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. It ensures that the signature is valid for the
// enclosed payload. As a policy, the registry only tries to store valid
-// content, leaving trust policies of that content up to consumems.
+// content, leaving trust policies of that content up to consumers.
func (ms *signedManifestHandler) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest, skipDependencyVerification bool) error {
var errs distribution.ErrManifestVerification