From 9986e8ca7c41dbf3364b962f163a142be8b16841 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Thu, 17 Nov 2016 12:28:05 -0600 Subject: [PATCH 01/20] adds support for oci manifests and manifestlists Signed-off-by: Mike Brown --- manifest/manifestlist/manifestlist.go | 28 +- manifest/manifestlist/manifestlist_test.go | 103 ++++++- manifest/ocischema/builder.go | 80 ++++++ manifest/ocischema/builder_test.go | 210 ++++++++++++++ manifest/ocischema/manifest.go | 133 +++++++++ manifest/ocischema/manifest_test.go | 111 ++++++++ registry/handlers/api_test.go | 30 +- registry/handlers/manifests.go | 316 ++++++++++++++++++--- registry/storage/driver/storagedriver.go | 2 +- registry/storage/manifeststore.go | 6 +- 10 files changed, 957 insertions(+), 62 deletions(-) create mode 100644 manifest/ocischema/builder.go create mode 100644 manifest/ocischema/builder_test.go create mode 100644 manifest/ocischema/manifest.go create mode 100644 manifest/ocischema/manifest_test.go diff --git a/manifest/manifestlist/manifestlist.go b/manifest/manifestlist/manifestlist.go index 3aa0662d..56f6aa61 100644 --- a/manifest/manifestlist/manifestlist.go +++ b/manifest/manifestlist/manifestlist.go @@ -7,11 +7,17 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest" + "github.com/docker/distribution/manifest/ocischema" "github.com/opencontainers/go-digest" ) -// MediaTypeManifestList specifies the mediaType for manifest lists. -const MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" +const ( + // MediaTypeManifestList specifies the mediaType for manifest lists. + MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" + // MediaTypeOCIManifestList specifies the mediaType for OCI compliant manifest + // lists. + MediaTypeOCIManifestList = "application/vnd.oci.image.manifest.list.v1+json" +) // SchemaVersion provides a pre-initialized version structure for this // packages version of the manifest. @@ -20,6 +26,13 @@ var SchemaVersion = manifest.Versioned{ MediaType: MediaTypeManifestList, } +// OCISchemaVersion provides a pre-initialized version structure for this +// packages OCIschema version of the manifest. +var OCISchemaVersion = manifest.Versioned{ + SchemaVersion: 2, + MediaType: MediaTypeOCIManifestList, +} + func init() { manifestListFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { m := new(DeserializedManifestList) @@ -105,8 +118,15 @@ type DeserializedManifestList struct { // DeserializedManifestList which contains the resulting manifest list // and its JSON representation. func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) { - m := ManifestList{ - Versioned: SchemaVersion, + var m ManifestList + if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == ocischema.MediaTypeManifest { + m = ManifestList{ + Versioned: OCISchemaVersion, + } + } else { + m = ManifestList{ + Versioned: SchemaVersion, + } } m.Manifests = make([]ManifestDescriptor, len(descriptors), len(descriptors)) diff --git a/manifest/manifestlist/manifestlist_test.go b/manifest/manifestlist/manifestlist_test.go index 09e6ed1f..1d24aef2 100644 --- a/manifest/manifestlist/manifestlist_test.go +++ b/manifest/manifestlist/manifestlist_test.go @@ -69,7 +69,7 @@ func TestManifestList(t *testing.T) { t.Fatalf("error creating DeserializedManifestList: %v", err) } - mediaType, canonical, err := deserialized.Payload() + mediaType, canonical, _ := deserialized.Payload() if mediaType != MediaTypeManifestList { t.Fatalf("unexpected media type: %s", mediaType) @@ -109,3 +109,104 @@ func TestManifestList(t *testing.T) { } } } + +var expectedOCIManifestListSerialization = []byte(`{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.list.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 985, + "digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", + "platform": { + "architecture": "amd64", + "os": "linux", + "features": [ + "sse4" + ] + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 2392, + "digest": "sha256:6346340964309634683409684360934680934608934608934608934068934608", + "platform": { + "architecture": "sun4m", + "os": "sunos" + } + } + ] +}`) + +func TestOCIManifestList(t *testing.T) { + manifestDescriptors := []ManifestDescriptor{ + { + Descriptor: distribution.Descriptor{ + Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", + Size: 985, + MediaType: "application/vnd.oci.image.manifest.v1+json", + }, + Platform: PlatformSpec{ + Architecture: "amd64", + OS: "linux", + Features: []string{"sse4"}, + }, + }, + { + Descriptor: distribution.Descriptor{ + Digest: "sha256:6346340964309634683409684360934680934608934608934608934068934608", + Size: 2392, + MediaType: "application/vnd.oci.image.manifest.v1+json", + }, + Platform: PlatformSpec{ + Architecture: "sun4m", + OS: "sunos", + }, + }, + } + + deserialized, err := FromDescriptors(manifestDescriptors) + if err != nil { + t.Fatalf("error creating DeserializedManifestList: %v", err) + } + + mediaType, canonical, _ := deserialized.Payload() + + if mediaType != MediaTypeOCIManifestList { + t.Fatalf("unexpected media type: %s", mediaType) + } + + // Check that the canonical field is the same as json.MarshalIndent + // with these parameters. + p, err := json.MarshalIndent(&deserialized.ManifestList, "", " ") + if err != nil { + t.Fatalf("error marshaling manifest list: %v", err) + } + if !bytes.Equal(p, canonical) { + t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(p)) + } + + // Check that the canonical field has the expected value. + if !bytes.Equal(expectedOCIManifestListSerialization, canonical) { + t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedOCIManifestListSerialization)) + } + + var unmarshalled DeserializedManifestList + if err := json.Unmarshal(deserialized.canonical, &unmarshalled); err != nil { + t.Fatalf("error unmarshaling manifest: %v", err) + } + + if !reflect.DeepEqual(&unmarshalled, deserialized) { + t.Fatalf("manifests are different after unmarshaling: %v != %v", unmarshalled, *deserialized) + } + + references := deserialized.References() + if len(references) != 2 { + t.Fatalf("unexpected number of references: %d", len(references)) + } + for i := range references { + if !reflect.DeepEqual(references[i], manifestDescriptors[i].Descriptor) { + t.Fatalf("unexpected value %d returned by References: %v", i, references[i]) + } + } +} diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go new file mode 100644 index 00000000..5318c139 --- /dev/null +++ b/manifest/ocischema/builder.go @@ -0,0 +1,80 @@ +package ocischema + +import ( + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/opencontainers/go-digest" +) + +// builder is a type for constructing manifests. +type builder struct { + // bs is a BlobService used to publish the configuration blob. + bs distribution.BlobService + + // configJSON references + configJSON []byte + + // layers is a list of layer descriptors that gets built by successive + // calls to AppendReference. + layers []distribution.Descriptor +} + +// NewManifestBuilder is used to build new manifests for the current schema +// version. It takes a BlobService so it can publish the configuration blob +// as part of the Build process. +func NewManifestBuilder(bs distribution.BlobService, configJSON []byte) distribution.ManifestBuilder { + mb := &builder{ + bs: bs, + configJSON: make([]byte, len(configJSON)), + } + copy(mb.configJSON, configJSON) + + return mb +} + +// Build produces a final manifest from the given references. +func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { + m := Manifest{ + Versioned: SchemaVersion, + Layers: make([]distribution.Descriptor, len(mb.layers)), + } + copy(m.Layers, mb.layers) + + configDigest := digest.FromBytes(mb.configJSON) + + var err error + m.Config, err = mb.bs.Stat(ctx, configDigest) + switch err { + case nil: + // Override MediaType, since Put always replaces the specified media + // type with application/octet-stream in the descriptor it returns. + m.Config.MediaType = MediaTypeConfig + return FromStruct(m) + case distribution.ErrBlobUnknown: + // nop + default: + return nil, err + } + + // Add config to the blob store + m.Config, err = mb.bs.Put(ctx, MediaTypeConfig, mb.configJSON) + // Override MediaType, since Put always replaces the specified media + // type with application/octet-stream in the descriptor it returns. + m.Config.MediaType = MediaTypeConfig + if err != nil { + return nil, err + } + + return FromStruct(m) +} + +// AppendReference adds a reference to the current ManifestBuilder. +func (mb *builder) AppendReference(d distribution.Describable) error { + mb.layers = append(mb.layers, d.Descriptor()) + return nil +} + +// References returns the current references added to this builder. +func (mb *builder) References() []distribution.Descriptor { + return mb.layers +} diff --git a/manifest/ocischema/builder_test.go b/manifest/ocischema/builder_test.go new file mode 100644 index 00000000..7192ac59 --- /dev/null +++ b/manifest/ocischema/builder_test.go @@ -0,0 +1,210 @@ +package ocischema + +import ( + "reflect" + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/opencontainers/go-digest" +) + +type mockBlobService struct { + descriptors map[digest.Digest]distribution.Descriptor +} + +func (bs *mockBlobService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) { + if descriptor, ok := bs.descriptors[dgst]; ok { + return descriptor, nil + } + return distribution.Descriptor{}, distribution.ErrBlobUnknown +} + +func (bs *mockBlobService) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) { + panic("not implemented") +} + +func (bs *mockBlobService) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) { + panic("not implemented") +} + +func (bs *mockBlobService) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) { + d := distribution.Descriptor{ + Digest: digest.FromBytes(p), + Size: int64(len(p)), + MediaType: "application/octet-stream", + } + bs.descriptors[d.Digest] = d + return d, nil +} + +func (bs *mockBlobService) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) { + panic("not implemented") +} + +func (bs *mockBlobService) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) { + panic("not implemented") +} + +func TestBuilder(t *testing.T) { + imgJSON := []byte(`{ + "architecture": "amd64", + "config": { + "AttachStderr": false, + "AttachStdin": false, + "AttachStdout": false, + "Cmd": [ + "/bin/sh", + "-c", + "echo hi" + ], + "Domainname": "", + "Entrypoint": null, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "derived=true", + "asdf=true" + ], + "Hostname": "23304fc829f9", + "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246", + "Labels": {}, + "OnBuild": [], + "OpenStdin": false, + "StdinOnce": false, + "Tty": false, + "User": "", + "Volumes": null, + "WorkingDir": "" + }, + "container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001", + "container_config": { + "AttachStderr": false, + "AttachStdin": false, + "AttachStdout": false, + "Cmd": [ + "/bin/sh", + "-c", + "#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]" + ], + "Domainname": "", + "Entrypoint": null, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "derived=true", + "asdf=true" + ], + "Hostname": "23304fc829f9", + "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246", + "Labels": {}, + "OnBuild": [], + "OpenStdin": false, + "StdinOnce": false, + "Tty": false, + "User": "", + "Volumes": null, + "WorkingDir": "" + }, + "created": "2015-11-04T23:06:32.365666163Z", + "docker_version": "1.9.0-dev", + "history": [ + { + "created": "2015-10-31T22:22:54.690851953Z", + "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /" + }, + { + "created": "2015-10-31T22:22:55.613815829Z", + "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]" + }, + { + "created": "2015-11-04T23:06:30.934316144Z", + "created_by": "/bin/sh -c #(nop) ENV derived=true", + "empty_layer": true + }, + { + "created": "2015-11-04T23:06:31.192097572Z", + "created_by": "/bin/sh -c #(nop) ENV asdf=true", + "empty_layer": true + }, + { + "created": "2015-11-04T23:06:32.083868454Z", + "created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024" + }, + { + "created": "2015-11-04T23:06:32.365666163Z", + "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]", + "empty_layer": true + } + ], + "os": "linux", + "rootfs": { + "diff_ids": [ + "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1", + "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef", + "sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49" + ], + "type": "layers" + } +}`) + configDigest := digest.FromBytes(imgJSON) + + descriptors := []distribution.Descriptor{ + { + Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"), + Size: 5312, + MediaType: MediaTypeLayer, + }, + { + Digest: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"), + Size: 235231, + MediaType: MediaTypeLayer, + }, + { + Digest: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"), + Size: 639152, + MediaType: MediaTypeLayer, + }, + } + + bs := &mockBlobService{descriptors: make(map[digest.Digest]distribution.Descriptor)} + builder := NewManifestBuilder(bs, imgJSON) + + for _, d := range descriptors { + if err := builder.AppendReference(d); err != nil { + t.Fatalf("AppendReference returned error: %v", err) + } + } + + built, err := builder.Build(context.Background()) + if err != nil { + t.Fatalf("Build returned error: %v", err) + } + + // Check that the config was put in the blob store + _, err = bs.Stat(context.Background(), configDigest) + if err != nil { + t.Fatal("config was not put in the blob store") + } + + manifest := built.(*DeserializedManifest).Manifest + + if manifest.Versioned.SchemaVersion != 2 { + t.Fatal("SchemaVersion != 2") + } + + target := manifest.Target() + if target.Digest != configDigest { + t.Fatalf("unexpected digest in target: %s", target.Digest.String()) + } + if target.MediaType != MediaTypeConfig { + t.Fatalf("unexpected media type in target: %s", target.MediaType) + } + if target.Size != 3153 { + t.Fatalf("unexpected size in target: %d", target.Size) + } + + references := manifest.References() + expected := append([]distribution.Descriptor{manifest.Target()}, descriptors...) + if !reflect.DeepEqual(references, expected) { + t.Fatal("References() does not match the descriptors added") + } +} diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go new file mode 100644 index 00000000..52f64678 --- /dev/null +++ b/manifest/ocischema/manifest.go @@ -0,0 +1,133 @@ +package ocischema + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/docker/distribution" + "github.com/docker/distribution/manifest" + "github.com/opencontainers/go-digest" +) + +const ( + // MediaTypeManifest specifies the mediaType for the current version. + MediaTypeManifest = "application/vnd.oci.image.manifest.v1+json" + + // MediaTypeConfig specifies the mediaType for the image configuration. + MediaTypeConfig = "application/vnd.oci.image.config.v1+json" + + // MediaTypePluginConfig specifies the mediaType for plugin configuration. + MediaTypePluginConfig = "application/vnd.docker.plugin.v1+json" + + // MediaTypeLayer is the mediaType used for layers referenced by the manifest. + MediaTypeLayer = "application/vnd.oci.image.layer.v1.tar+gzip" + + // MediaTypeForeignLayer is the mediaType used for layers that must be + // downloaded from foreign URLs. + MediaTypeForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" +) + +var ( + // SchemaVersion provides a pre-initialized version structure for this + // packages version of the manifest. + SchemaVersion = manifest.Versioned{ + SchemaVersion: 2, // Mike: todo this could confusing cause oci version 1 is closer to docker 2 than 1 + MediaType: MediaTypeManifest, + } +) + +func init() { + ocischemaFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + m := new(DeserializedManifest) + err := m.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + dgst := digest.FromBytes(b) + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifest}, err + } + err := distribution.RegisterManifestSchema(MediaTypeManifest, ocischemaFunc) + if err != nil { + panic(fmt.Sprintf("Unable to register manifest: %s", err)) + } +} + +// Manifest defines a schema2 manifest. +type Manifest struct { + manifest.Versioned + + // Config references the image configuration as a blob. + Config distribution.Descriptor `json:"config"` + + // Layers lists descriptors for the layers referenced by the + // configuration. + Layers []distribution.Descriptor `json:"layers"` +} + +// References returnes the descriptors of this manifests references. +func (m Manifest) References() []distribution.Descriptor { + references := make([]distribution.Descriptor, 0, 1+len(m.Layers)) + references = append(references, m.Config) + references = append(references, m.Layers...) + return references +} + +// Target returns the target of this signed manifest. +func (m Manifest) Target() distribution.Descriptor { + return m.Config +} + +// DeserializedManifest wraps Manifest with a copy of the original JSON. +// It satisfies the distribution.Manifest interface. +type DeserializedManifest struct { + Manifest + + // canonical is the canonical byte representation of the Manifest. + canonical []byte +} + +// FromStruct takes a Manifest structure, marshals it to JSON, and returns a +// DeserializedManifest which contains the manifest and its JSON representation. +func FromStruct(m Manifest) (*DeserializedManifest, error) { + var deserialized DeserializedManifest + deserialized.Manifest = m + + var err error + deserialized.canonical, err = json.MarshalIndent(&m, "", " ") + return &deserialized, err +} + +// UnmarshalJSON populates a new Manifest struct from JSON data. +func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { + m.canonical = make([]byte, len(b), len(b)) + // store manifest in canonical + copy(m.canonical, b) + + // Unmarshal canonical JSON into Manifest object + var manifest Manifest + if err := json.Unmarshal(m.canonical, &manifest); err != nil { + return err + } + + m.Manifest = manifest + + return nil +} + +// MarshalJSON returns the contents of canonical. If canonical is empty, +// marshals the inner contents. +func (m *DeserializedManifest) MarshalJSON() ([]byte, error) { + if len(m.canonical) > 0 { + return m.canonical, nil + } + + return nil, errors.New("JSON representation not initialized in DeserializedManifest") +} + +// Payload returns the raw content of the manifest. The contents can be used to +// calculate the content identifier. +func (m DeserializedManifest) Payload() (string, []byte, error) { + return m.MediaType, m.canonical, nil +} diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go new file mode 100644 index 00000000..9f439dcd --- /dev/null +++ b/manifest/ocischema/manifest_test.go @@ -0,0 +1,111 @@ +package ocischema + +import ( + "bytes" + "encoding/json" + "reflect" + "testing" + + "github.com/docker/distribution" +) + +var expectedManifestSerialization = []byte(`{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": 985, + "digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": 153263, + "digest": "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b" + } + ] +}`) + +func TestManifest(t *testing.T) { + manifest := Manifest{ + Versioned: SchemaVersion, + Config: distribution.Descriptor{ + Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", + Size: 985, + MediaType: MediaTypeConfig, + }, + Layers: []distribution.Descriptor{ + { + Digest: "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b", + Size: 153263, + MediaType: MediaTypeLayer, + }, + }, + } + + deserialized, err := FromStruct(manifest) + if err != nil { + t.Fatalf("error creating DeserializedManifest: %v", err) + } + + mediaType, canonical, err := deserialized.Payload() + + if mediaType != MediaTypeManifest { + t.Fatalf("unexpected media type: %s", mediaType) + } + + // Check that the canonical field is the same as json.MarshalIndent + // with these parameters. + p, err := json.MarshalIndent(&manifest, "", " ") + if err != nil { + t.Fatalf("error marshaling manifest: %v", err) + } + if !bytes.Equal(p, canonical) { + t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(p)) + } + + // Check that canonical field matches expected value. + if !bytes.Equal(expectedManifestSerialization, canonical) { + t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedManifestSerialization)) + } + + var unmarshalled DeserializedManifest + if err := json.Unmarshal(deserialized.canonical, &unmarshalled); err != nil { + t.Fatalf("error unmarshaling manifest: %v", err) + } + + if !reflect.DeepEqual(&unmarshalled, deserialized) { + t.Fatalf("manifests are different after unmarshaling: %v != %v", unmarshalled, *deserialized) + } + + target := deserialized.Target() + if target.Digest != "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" { + t.Fatalf("unexpected digest in target: %s", target.Digest.String()) + } + if target.MediaType != MediaTypeConfig { + t.Fatalf("unexpected media type in target: %s", target.MediaType) + } + if target.Size != 985 { + t.Fatalf("unexpected size in target: %d", target.Size) + } + + references := deserialized.References() + if len(references) != 2 { + t.Fatalf("unexpected number of references: %d", len(references)) + } + + if !reflect.DeepEqual(references[0], target) { + t.Fatalf("first reference should be target: %v != %v", references[0], target) + } + + // Test the second reference + if references[1].Digest != "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b" { + t.Fatalf("unexpected digest in reference: %s", references[0].Digest.String()) + } + if references[1].MediaType != MediaTypeLayer { + t.Fatalf("unexpected media type in reference: %s", references[0].MediaType) + } + if references[1].Size != 153263 { + t.Fatalf("unexpected size in reference: %d", references[0].Size) + } +} diff --git a/registry/handlers/api_test.go b/registry/handlers/api_test.go index 98dcaa71..446f1b83 100644 --- a/registry/handlers/api_test.go +++ b/registry/handlers/api_test.go @@ -478,7 +478,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv { // ----------------------------------------- // Do layer push with an empty body and different digest - uploadURLBase, uploadUUID = startPushLayer(t, env, imageName) + uploadURLBase, _ = startPushLayer(t, env, imageName) resp, err = doPushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, bytes.NewReader([]byte{})) if err != nil { t.Fatalf("unexpected error doing bad layer push: %v", err) @@ -494,7 +494,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv { t.Fatalf("unexpected error digesting empty buffer: %v", err) } - uploadURLBase, uploadUUID = startPushLayer(t, env, imageName) + uploadURLBase, _ = startPushLayer(t, env, imageName) pushLayer(t, env.builder, imageName, zeroDigest, uploadURLBase, bytes.NewReader([]byte{})) // ----------------------------------------- @@ -507,7 +507,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv { t.Fatalf("unexpected error digesting empty tar: %v", err) } - uploadURLBase, uploadUUID = startPushLayer(t, env, imageName) + uploadURLBase, _ = startPushLayer(t, env, imageName) pushLayer(t, env.builder, imageName, emptyDigest, uploadURLBase, bytes.NewReader(emptyTar)) // ------------------------------------------ @@ -515,7 +515,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv { layerLength, _ := layerFile.Seek(0, os.SEEK_END) layerFile.Seek(0, os.SEEK_SET) - uploadURLBase, uploadUUID = startPushLayer(t, env, imageName) + uploadURLBase, _ = startPushLayer(t, env, imageName) pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile) // ------------------------------------------ @@ -529,7 +529,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv { canonicalDigest := canonicalDigester.Digest() layerFile.Seek(0, 0) - uploadURLBase, uploadUUID = startPushLayer(t, env, imageName) + uploadURLBase, _ = startPushLayer(t, env, imageName) uploadURLBase, dgst := pushChunk(t, env.builder, imageName, uploadURLBase, layerFile, layerLength) finishUpload(t, env.builder, imageName, uploadURLBase, dgst) @@ -612,7 +612,7 @@ func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv { t.Fatalf("Error constructing request: %s", err) } req.Header.Set("If-None-Match", "") - resp, err = http.DefaultClient.Do(req) + resp, _ = http.DefaultClient.Do(req) checkResponse(t, "fetching layer with invalid etag", resp, http.StatusOK) // Missing tests: @@ -1874,7 +1874,7 @@ func testManifestDelete(t *testing.T, env *testEnv, args manifestArgs) { manifest := args.manifest ref, _ := reference.WithDigest(imageName, dgst) - manifestDigestURL, err := env.builder.BuildManifestURL(ref) + manifestDigestURL, _ := env.builder.BuildManifestURL(ref) // --------------- // Delete by digest resp, err := httpDelete(manifestDigestURL) @@ -1935,7 +1935,7 @@ func testManifestDelete(t *testing.T, env *testEnv, args manifestArgs) { // Upload manifest by tag tag := "atag" tagRef, _ := reference.WithTag(imageName, tag) - manifestTagURL, err := env.builder.BuildManifestURL(tagRef) + manifestTagURL, _ := env.builder.BuildManifestURL(tagRef) resp = putManifest(t, "putting manifest by tag", manifestTagURL, args.mediaType, manifest) checkResponse(t, "putting manifest by tag", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ @@ -2502,7 +2502,7 @@ func TestRegistryAsCacheMutationAPIs(t *testing.T) { checkResponse(t, "putting signed manifest to cache", resp, errcode.ErrorCodeUnsupported.Descriptor().HTTPStatusCode) // Manifest Delete - resp, err = httpDelete(manifestURL) + resp, _ = httpDelete(manifestURL) checkResponse(t, "deleting signed manifest from cache", resp, errcode.ErrorCodeUnsupported.Descriptor().HTTPStatusCode) // Blob upload initialization @@ -2521,8 +2521,8 @@ func TestRegistryAsCacheMutationAPIs(t *testing.T) { // Blob Delete ref, _ := reference.WithDigest(imageName, digestSha256EmptyTar) - blobURL, err := env.builder.BuildBlobURL(ref) - resp, err = httpDelete(blobURL) + blobURL, _ := env.builder.BuildBlobURL(ref) + resp, _ = httpDelete(blobURL) checkResponse(t, "deleting blob from cache", resp, errcode.ErrorCodeUnsupported.Descriptor().HTTPStatusCode) } @@ -2601,9 +2601,9 @@ func TestProxyManifestGetByTag(t *testing.T) { checkErr(t, err, "building manifest url") resp, err = http.Get(manifestTagURL) - checkErr(t, err, "fetching manifest from proxy by tag") + checkErr(t, err, "fetching manifest from proxy by tag (error check 1)") defer resp.Body.Close() - checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK) + checkResponse(t, "fetching manifest from proxy by tag (response check 1)", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{dgst.String()}, }) @@ -2616,9 +2616,9 @@ func TestProxyManifestGetByTag(t *testing.T) { // fetch it with the same proxy URL as before. Ensure the updated content is at the same tag resp, err = http.Get(manifestTagURL) - checkErr(t, err, "fetching manifest from proxy by tag") + checkErr(t, err, "fetching manifest from proxy by tag (error check 2)") defer resp.Body.Close() - checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK) + checkResponse(t, "fetching manifest from proxy by tag (response check 2)", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{newDigest.String()}, }) diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index 4e1d9868..af0a8876 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -9,6 +9,7 @@ import ( "github.com/docker/distribution" dcontext "github.com/docker/distribution/context" "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/docker/distribution/reference" @@ -72,43 +73,10 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) imh.Errors = append(imh.Errors, err) return } - - var manifest distribution.Manifest - if imh.Tag != "" { - tags := imh.Repository.Tags(imh) - desc, err := tags.Get(imh, imh.Tag) - if err != nil { - if _, ok := err.(distribution.ErrTagUnknown); ok { - imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) - } else { - imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) - } - return - } - imh.Digest = desc.Digest - } - - if etagMatch(r, imh.Digest.String()) { - w.WriteHeader(http.StatusNotModified) - return - } - - var options []distribution.ManifestServiceOption - if imh.Tag != "" { - options = append(options, distribution.WithTag(imh.Tag)) - } - manifest, err = manifests.Get(imh, imh.Digest, options...) - if err != nil { - if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { - imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) - } else { - imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) - } - return - } - supportsSchema2 := false supportsManifestList := false + supportsOCISchema := false + supportsOCIManifestList := false // this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values // https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202 for _, acceptHeader := range r.Header["Accept"] { @@ -132,16 +100,259 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) if mediaType == manifestlist.MediaTypeManifestList { supportsManifestList = true } + if mediaType == ocischema.MediaTypeManifest { + supportsOCISchema = true + } + if mediaType == manifestlist.MediaTypeOCIManifestList { + supportsOCIManifestList = true + } } } + supportsOCI := supportsOCISchema || supportsOCIManifestList + + var manifest distribution.Manifest + if imh.Tag != "" { + tags := imh.Repository.Tags(imh) + var desc distribution.Descriptor + if !supportsOCI { + desc, err = tags.Get(imh, imh.Tag) + } else { + desc, err = tags.Get(imh, imh.annotatedTag(false)) + } + if err != nil { + if _, ok := err.(distribution.ErrTagUnknown); ok { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + } else { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + imh.Digest = desc.Digest + } + + if etagMatch(r, imh.Digest.String()) { + w.WriteHeader(http.StatusNotModified) + return + } + + var options []distribution.ManifestServiceOption + if imh.Tag != "" { + options = append(options, distribution.WithTag(imh.annotatedTag(supportsOCI))) + } + manifest, err = manifests.Get(imh, imh.Digest, options...) + if err != nil { + if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + } else { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest) manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList) + isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == ocischema.MediaTypeManifest) + isAnOCIManifestList := isManifestList && (manifestList.MediaType == manifestlist.MediaTypeOCIManifestList) + + if (isSchema2 && !isAnOCIManifest) && (supportsOCISchema && !supportsSchema2) { + fmt.Printf("\n\nmanifest is schema2, but accept header only supports OCISchema \n\n") + w.WriteHeader(http.StatusNotFound) + return + } + if (isManifestList && !isAnOCIManifestList) && (supportsOCIManifestList && !supportsManifestList) { + fmt.Printf("\n\nmanifestlist is not OCI, but accept header only supports an OCI manifestlist\n\n") + w.WriteHeader(http.StatusNotFound) + return + } + if isAnOCIManifest && (!supportsOCISchema && supportsSchema2) { + fmt.Printf("\n\nmanifest is OCI, but accept header only supports schema2\n\n") + w.WriteHeader(http.StatusNotFound) + return + } + if isAnOCIManifestList && (!supportsOCIManifestList && supportsManifestList) { + fmt.Printf("\n\nmanifestlist is OCI, but accept header only supports non-OCI manifestlists\n\n") + w.WriteHeader(http.StatusNotFound) + return + } + // Only rewrite schema2 manifests when they are being fetched by tag. + // If they are being fetched by digest, we can't return something not + // matching the digest. + if imh.Tag != "" && isSchema2 && !(supportsSchema2 || supportsOCISchema) { + // Rewrite manifest in schema1 format + ctxu.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String()) + + manifest, err = imh.convertSchema2Manifest(schema2Manifest) + if err != nil { + return + } + } else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIManifestList) { + // Rewrite manifest in schema1 format + ctxu.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) + + // Find the image manifest corresponding to the default + // platform + var manifestDigest digest.Digest + for _, manifestDescriptor := range manifestList.Manifests { + if manifestDescriptor.Platform.Architecture == defaultArch && manifestDescriptor.Platform.OS == defaultOS { + manifestDigest = manifestDescriptor.Digest + break + } + } + + if manifestDigest == "" { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown) + return + } + + manifest, err = manifests.Get(imh, manifestDigest) + if err != nil { + if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + } else { + imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) + } + return + } + + // If necessary, convert the image manifest + if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !(supportsSchema2 || supportsOCISchema) { + manifest, err = imh.convertSchema2Manifest(schema2Manifest) + if err != nil { + return + } + } else { + imh.Digest = manifestDigest + } + } + + ct, p, err := manifest.Payload() + if err != nil { + return + } + + w.Header().Set("Content-Type", ct) + w.Header().Set("Content-Length", fmt.Sprint(len(p))) + w.Header().Set("Docker-Content-Digest", imh.Digest.String()) + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest)) + w.Write(p) +} + +// GetImageManifest fetches the image manifest from the storage backend, if it exists. +func (imh *manifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) { + fmt.Printf("\n\nGetting a manifest!\n\n\n") + supportsSchema2 := false + supportsManifestList := false + supportsOCISchema := false + supportsOCIManifestList := false + + // this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values + // https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202 + for _, acceptHeader := range r.Header["Accept"] { + // r.Header[...] is a slice in case the request contains the same header more than once + // if the header isn't set, we'll get the zero value, which "range" will handle gracefully + + // we need to split each header value on "," to get the full list of "Accept" values (per RFC 2616) + // https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1 + for _, mediaType := range strings.Split(acceptHeader, ",") { + // remove "; q=..." if present + if i := strings.Index(mediaType, ";"); i >= 0 { + mediaType = mediaType[:i] + } + + // it's common (but not required) for Accept values to be space separated ("a/b, c/d, e/f") + mediaType = strings.TrimSpace(mediaType) + + if mediaType == schema2.MediaTypeManifest { + supportsSchema2 = true + } + if mediaType == manifestlist.MediaTypeManifestList { + supportsManifestList = true + } + if mediaType == ocischema.MediaTypeManifest { + supportsOCISchema = true + } + if mediaType == manifestlist.MediaTypeOCIManifestList { + supportsOCIManifestList = true + } + } + } + supportsOCI := supportsOCISchema || supportsOCIManifestList + + ctxu.GetLogger(imh).Debug("GetImageManifest") + manifests, err := imh.Repository.Manifests(imh) + if err != nil { + imh.Errors = append(imh.Errors, err) + return + } + + var manifest distribution.Manifest + if imh.Tag != "" { + tags := imh.Repository.Tags(imh) + var desc distribution.Descriptor + if !supportsOCI { + desc, err = tags.Get(imh, imh.Tag) + if err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + return + } + } else { + desc, err = tags.Get(imh, imh.annotatedTag(supportsOCI)) + if err != nil { + desc, err = tags.Get(imh, imh.annotatedTag(false)) + if err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + return + } + } + } + imh.Digest = desc.Digest + } + + if etagMatch(r, imh.Digest.String()) { + w.WriteHeader(http.StatusNotModified) + return + } + + var options []distribution.ManifestServiceOption + if imh.Tag != "" { + options = append(options, distribution.WithTag(imh.annotatedTag(supportsOCI))) + } + manifest, err = manifests.Get(imh, imh.Digest, options...) + if err != nil { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) + return + } + + schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest) + manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList) + isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == ocischema.MediaTypeManifest) + isAnOCIManifestList := isManifestList && (manifestList.MediaType == manifestlist.MediaTypeOCIManifestList) + + badCombinations := [][]bool{ + {isSchema2 && !isAnOCIManifest, supportsOCISchema && !supportsSchema2}, + {isManifestList && !isAnOCIManifestList, supportsOCIManifestList && !supportsManifestList}, + {isAnOCIManifest, !supportsOCISchema && supportsSchema2}, + {isAnOCIManifestList, !supportsOCIManifestList && supportsManifestList}, + } + for i, combo := range badCombinations { + if combo[0] && combo[1] { + fmt.Printf("\n\nbad combo! %d\n\n\n", i) + w.WriteHeader(http.StatusNotFound) + return + } + } + if isAnOCIManifest { + fmt.Print("\n\nreturning OCI manifest\n\n") + } else if isSchema2 { + fmt.Print("\n\nreturning schema 2 manifest\n\n") + } else { + fmt.Print("\n\nreturning schema 1 manifest\n\n") + } // Only rewrite schema2 manifests when they are being fetched by tag. // If they are being fetched by digest, we can't return something not // matching the digest. - if imh.Tag != "" && isSchema2 && !supportsSchema2 { + if imh.Tag != "" && isSchema2 && !(supportsSchema2 || supportsOCISchema) { // Rewrite manifest in schema1 format dcontext.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String()) @@ -149,7 +360,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) if err != nil { return } - } else if imh.Tag != "" && isManifestList && !supportsManifestList { + } else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIManifestList) { // Rewrite manifest in schema1 format dcontext.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) @@ -199,6 +410,8 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) w.Header().Set("Docker-Content-Digest", imh.Digest.String()) w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest)) w.Write(p) + + fmt.Printf("\n\nSucceeded in getting the manifest!\n\n\n") } func (imh *manifestHandler) convertSchema2Manifest(schema2Manifest *schema2.DeserializedManifest) (distribution.Manifest, error) { @@ -286,9 +499,17 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) return } + isAnOCIManifest := mediaType == ocischema.MediaTypeManifest || mediaType == manifestlist.MediaTypeOCIManifestList + + if isAnOCIManifest { + fmt.Printf("\n\nPutting an OCI Manifest!\n\n\n") + } else { + fmt.Printf("\n\nPutting a Docker Manifest!\n\n\n") + } + var options []distribution.ManifestServiceOption if imh.Tag != "" { - options = append(options, distribution.WithTag(imh.Tag)) + options = append(options, distribution.WithTag(imh.annotatedTag(isAnOCIManifest))) } if err := imh.applyResourcePolicy(manifest); err != nil { @@ -301,10 +522,12 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) // TODO(stevvooe): These error handling switches really need to be // handled by an app global mapper. if err == distribution.ErrUnsupported { + fmt.Printf("\n\nXXX 1\n\n\n") imh.Errors = append(imh.Errors, errcode.ErrorCodeUnsupported) return } if err == distribution.ErrAccessDenied { + fmt.Printf("\n\nXXX 2\n\n\n") imh.Errors = append(imh.Errors, errcode.ErrorCodeDenied) return } @@ -331,15 +554,16 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) default: imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) } - + fmt.Printf("\n\nXXX 3\n\n\n") return } // Tag this manifest if imh.Tag != "" { tags := imh.Repository.Tags(imh) - err = tags.Tag(imh, imh.Tag, desc) + err = tags.Tag(imh, imh.annotatedTag(isAnOCIManifest), desc) if err != nil { + fmt.Printf("\n\nXXX 4: %T: %v\n\n\n", err, err) imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return } @@ -349,6 +573,7 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) // Construct a canonical url for the uploaded manifest. ref, err := reference.WithDigest(imh.Repository.Named(), imh.Digest) if err != nil { + fmt.Printf("\n\nXXX 5\n\n\n") imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return } @@ -364,6 +589,8 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) w.Header().Set("Location", location) w.Header().Set("Docker-Content-Digest", imh.Digest.String()) w.WriteHeader(http.StatusCreated) + + fmt.Printf("\n\nSucceeded in putting manifest!\n\n\n") } // applyResourcePolicy checks whether the resource class matches what has @@ -478,3 +705,12 @@ func (imh *manifestHandler) DeleteManifest(w http.ResponseWriter, r *http.Reques w.WriteHeader(http.StatusAccepted) } + +// annotatedTag will annotate OCI tags by prepending a string, and leave docker +// tags unmodified. +func (imh *manifestHandler) annotatedTag(oci bool) string { + if oci { + return "oci." + imh.Tag + } + return imh.Tag +} diff --git a/registry/storage/driver/storagedriver.go b/registry/storage/driver/storagedriver.go index b220713f..46c1b9e0 100644 --- a/registry/storage/driver/storagedriver.go +++ b/registry/storage/driver/storagedriver.go @@ -116,7 +116,7 @@ type FileWriter interface { // number of path components separated by slashes, where each component is // restricted to alphanumeric characters or a period, underscore, or // hyphen. -var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._-]+)+$`) +var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._:-]+)+$`) // ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method. type ErrUnsupportedMethod struct { diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 20e34eb0..52bbe9da 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -9,6 +9,7 @@ import ( dcontext "github.com/docker/distribution/context" "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/opencontainers/go-digest" @@ -48,6 +49,7 @@ type manifestStore struct { schema1Handler ManifestHandler schema2Handler ManifestHandler + ocischemaHandler ManifestHandler manifestListHandler ManifestHandler } @@ -99,7 +101,9 @@ func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options .. switch versioned.MediaType { case schema2.MediaTypeManifest: return ms.schema2Handler.Unmarshal(ctx, dgst, content) - case manifestlist.MediaTypeManifestList: + case ocischema.MediaTypeManifest: + return ms.ocischemaHandler.Unmarshal(ctx, dgst, content) + case manifestlist.MediaTypeManifestList, manifestlist.MediaTypeOCIManifestList: return ms.manifestListHandler.Unmarshal(ctx, dgst, content) default: return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)} From 6fcea22b0aa01f513393033e268913dea7fef180 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Mon, 10 Jul 2017 18:09:10 -0500 Subject: [PATCH 02/20] add an ocischema manifest handler for the registry Signed-off-by: Mike Brown --- registry/storage/manifeststore.go | 2 + registry/storage/ocimanifesthandler.go | 128 ++++++++++++++++++ registry/storage/ocimanifesthandler_test.go | 136 ++++++++++++++++++++ registry/storage/registry.go | 6 + 4 files changed, 272 insertions(+) create mode 100644 registry/storage/ocimanifesthandler.go create mode 100644 registry/storage/ocimanifesthandler_test.go diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 52bbe9da..13b4f5b5 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -121,6 +121,8 @@ func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest return ms.schema1Handler.Put(ctx, manifest, ms.skipDependencyVerification) case *schema2.DeserializedManifest: return ms.schema2Handler.Put(ctx, manifest, ms.skipDependencyVerification) + case *ocischema.DeserializedManifest: + return ms.ocischemaHandler.Put(ctx, manifest, ms.skipDependencyVerification) case *manifestlist.DeserializedManifestList: return ms.manifestListHandler.Put(ctx, manifest, ms.skipDependencyVerification) } diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go new file mode 100644 index 00000000..bb658348 --- /dev/null +++ b/registry/storage/ocimanifesthandler.go @@ -0,0 +1,128 @@ +package storage + +import ( + "encoding/json" + "fmt" + "net/url" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest/ocischema" + "github.com/opencontainers/go-digest" +) + +//ocischemaManifestHandler is a ManifestHandler that covers ocischema manifests. +type ocischemaManifestHandler struct { + repository distribution.Repository + blobStore distribution.BlobStore + ctx context.Context + manifestURLs manifestURLs +} + +var _ ManifestHandler = &ocischemaManifestHandler{} + +func (ms *ocischemaManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { + context.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Unmarshal") + + var m ocischema.DeserializedManifest + if err := json.Unmarshal(content, &m); err != nil { + return nil, err + } + + return &m, nil +} + +func (ms *ocischemaManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { + context.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Put") + + m, ok := manifest.(*ocischema.DeserializedManifest) + if !ok { + return "", fmt.Errorf("non-ocischema manifest put to ocischemaManifestHandler: %T", manifest) + } + + if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil { + return "", err + } + + mt, payload, err := m.Payload() + if err != nil { + return "", err + } + + revision, err := ms.blobStore.Put(ctx, mt, payload) + if err != nil { + context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + return "", err + } + + return revision.Digest, nil +} + +// verifyManifest ensures that the manifest content is valid from the +// perspective of the registry. As a policy, the registry only tries to store +// valid content, leaving trust policies of that content up to consumers. +func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst ocischema.DeserializedManifest, skipDependencyVerification bool) error { + var errs distribution.ErrManifestVerification + + if skipDependencyVerification { + return nil + } + + manifestService, err := ms.repository.Manifests(ctx) + if err != nil { + return err + } + + blobsService := ms.repository.Blobs(ctx) + + for _, descriptor := range mnfst.References() { + var err error + + switch descriptor.MediaType { + case ocischema.MediaTypeForeignLayer: + // Clients download this layer from an external URL, so do not check for + // its presense. + if len(descriptor.URLs) == 0 { + err = errMissingURL + } + allow := ms.manifestURLs.allow + deny := ms.manifestURLs.deny + for _, u := range descriptor.URLs { + var pu *url.URL + pu, err = url.Parse(u) + if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" || (allow != nil && !allow.MatchString(u)) || (deny != nil && deny.MatchString(u)) { + err = errInvalidURL + break + } + } + case ocischema.MediaTypeManifest: + var exists bool + exists, err = manifestService.Exists(ctx, descriptor.Digest) + if err != nil || !exists { + err = distribution.ErrBlobUnknown // just coerce to unknown. + } + + fallthrough // double check the blob store. + default: + // forward all else to blob storage + if len(descriptor.URLs) == 0 { + _, err = blobsService.Stat(ctx, descriptor.Digest) + } + } + + if err != nil { + if err != distribution.ErrBlobUnknown { + errs = append(errs, err) + } + + // On error here, we always append unknown blob errors. + errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: descriptor.Digest}) + } + } + + if len(errs) != 0 { + return errs + } + + return nil +} diff --git a/registry/storage/ocimanifesthandler_test.go b/registry/storage/ocimanifesthandler_test.go new file mode 100644 index 00000000..e09f1ee3 --- /dev/null +++ b/registry/storage/ocimanifesthandler_test.go @@ -0,0 +1,136 @@ +package storage + +import ( + "regexp" + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest" + "github.com/docker/distribution/manifest/ocischema" + "github.com/docker/distribution/registry/storage/driver/inmemory" +) + +func TestVerifyOCIManifestForeignLayer(t *testing.T) { + ctx := context.Background() + inmemoryDriver := inmemory.New() + registry := createRegistry(t, inmemoryDriver, + ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")), + ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope"))) + repo := makeRepository(t, registry, "test") + manifestService := makeManifestService(t, repo) + + config, err := repo.Blobs(ctx).Put(ctx, ocischema.MediaTypeConfig, nil) + if err != nil { + t.Fatal(err) + } + + layer, err := repo.Blobs(ctx).Put(ctx, ocischema.MediaTypeLayer, nil) + if err != nil { + t.Fatal(err) + } + + foreignLayer := distribution.Descriptor{ + Digest: "sha256:463435349086340864309863409683460843608348608934092322395278926a", + Size: 6323, + MediaType: ocischema.MediaTypeForeignLayer, + } + + template := ocischema.Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: ocischema.MediaTypeManifest, + }, + Config: config, + } + + type testcase struct { + BaseLayer distribution.Descriptor + URLs []string + Err error + } + + cases := []testcase{ + { + foreignLayer, + nil, + errMissingURL, + }, + { + // regular layers may have foreign urls + layer, + []string{"http://foo/bar"}, + nil, + }, + { + foreignLayer, + []string{"file:///local/file"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://foo/bar#baz"}, + errInvalidURL, + }, + { + foreignLayer, + []string{""}, + errInvalidURL, + }, + { + foreignLayer, + []string{"https://foo/bar", ""}, + errInvalidURL, + }, + { + foreignLayer, + []string{"", "https://foo/bar"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://nope/bar"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://foo/nope"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://foo/bar"}, + nil, + }, + { + foreignLayer, + []string{"https://foo/bar"}, + nil, + }, + } + + for _, c := range cases { + m := template + l := c.BaseLayer + l.URLs = c.URLs + m.Layers = []distribution.Descriptor{l} + dm, err := ocischema.FromStruct(m) + if err != nil { + t.Error(err) + continue + } + + _, err = manifestService.Put(ctx, dm) + if verr, ok := err.(distribution.ErrManifestVerification); ok { + // Extract the first error + if len(verr) == 2 { + if _, ok = verr[1].(distribution.ErrManifestBlobUnknown); ok { + err = verr[0] + } + } + } + if err != c.Err { + t.Errorf("%#v: expected %v, got %v", l, c.Err, err) + } + } +} diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 46b96853..f7c95a59 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -258,6 +258,12 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M repository: repo, blobStore: blobStore, }, + ocischemaHandler: &ocischemaManifestHandler{ + ctx: ctx, + repository: repo, + blobStore: blobStore, + manifestURLs: repo.registry.manifestURLs, + }, } // Apply options From c94f28805e1b653204cc4cec2d76a12398e4524a Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Tue, 11 Jul 2017 14:19:47 -0500 Subject: [PATCH 03/20] OCI media types; annotation support; oci index Signed-off-by: Mike Brown --- blobs.go | 8 ++++ manifest/manifestlist/manifestlist.go | 12 ++--- manifest/manifestlist/manifestlist_test.go | 13 +++--- manifest/ocischema/builder.go | 7 +-- manifest/ocischema/builder_test.go | 9 ++-- manifest/ocischema/manifest.go | 33 ++++--------- manifest/ocischema/manifest_test.go | 13 +++--- registry/handlers/manifests.go | 51 ++++++++++++--------- registry/storage/manifeststore.go | 5 +- registry/storage/ocimanifesthandler.go | 6 ++- registry/storage/ocimanifesthandler_test.go | 35 +++++++------- 11 files changed, 102 insertions(+), 90 deletions(-) diff --git a/blobs.go b/blobs.go index 145b0785..42a43773 100644 --- a/blobs.go +++ b/blobs.go @@ -10,6 +10,7 @@ import ( "github.com/docker/distribution/reference" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) var ( @@ -72,6 +73,13 @@ type Descriptor struct { // URLs contains the source URLs of this content. URLs []string `json:"urls,omitempty"` + // Annotations contains arbitrary metadata relating to the targeted content. + Annotations map[string]string `json:"annotations,omitempty"` + + // Platform describes the platform which the image in the manifest runs on. + // This should only be used when referring to a manifest. + Platform *v1.Platform `json:"platform,omitempty"` + // NOTE: Before adding a field here, please ensure that all // other options have been exhausted. Much of the type relationships // depend on the simplicity of this type. diff --git a/manifest/manifestlist/manifestlist.go b/manifest/manifestlist/manifestlist.go index 56f6aa61..3303c6be 100644 --- a/manifest/manifestlist/manifestlist.go +++ b/manifest/manifestlist/manifestlist.go @@ -7,16 +7,13 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest" - "github.com/docker/distribution/manifest/ocischema" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) const ( // MediaTypeManifestList specifies the mediaType for manifest lists. MediaTypeManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" - // MediaTypeOCIManifestList specifies the mediaType for OCI compliant manifest - // lists. - MediaTypeOCIManifestList = "application/vnd.oci.image.manifest.list.v1+json" ) // SchemaVersion provides a pre-initialized version structure for this @@ -30,7 +27,7 @@ var SchemaVersion = manifest.Versioned{ // packages OCIschema version of the manifest. var OCISchemaVersion = manifest.Versioned{ SchemaVersion: 2, - MediaType: MediaTypeOCIManifestList, + MediaType: v1.MediaTypeImageIndex, } func init() { @@ -92,6 +89,9 @@ type ManifestList struct { // Config references the image configuration as a blob. Manifests []ManifestDescriptor `json:"manifests"` + + // Annotations contains arbitrary metadata for the image index. + Annotations map[string]string `json:"annotations,omitempty"` } // References returns the distribution descriptors for the referenced image @@ -119,7 +119,7 @@ type DeserializedManifestList struct { // and its JSON representation. func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) { var m ManifestList - if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == ocischema.MediaTypeManifest { + if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == v1.MediaTypeImageManifest { m = ManifestList{ Versioned: OCISchemaVersion, } diff --git a/manifest/manifestlist/manifestlist_test.go b/manifest/manifestlist/manifestlist_test.go index 1d24aef2..5ab328dd 100644 --- a/manifest/manifestlist/manifestlist_test.go +++ b/manifest/manifestlist/manifestlist_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/docker/distribution" + "github.com/opencontainers/image-spec/specs-go/v1" ) var expectedManifestListSerialization = []byte(`{ @@ -110,9 +111,9 @@ func TestManifestList(t *testing.T) { } } -var expectedOCIManifestListSerialization = []byte(`{ +var expectedOCIImageIndexSerialization = []byte(`{ "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.manifest.list.v1+json", + "mediaType": "application/vnd.oci.image.index.v1+json", "manifests": [ { "mediaType": "application/vnd.oci.image.manifest.v1+json", @@ -138,7 +139,7 @@ var expectedOCIManifestListSerialization = []byte(`{ ] }`) -func TestOCIManifestList(t *testing.T) { +func TestOCIImageIndex(t *testing.T) { manifestDescriptors := []ManifestDescriptor{ { Descriptor: distribution.Descriptor{ @@ -172,7 +173,7 @@ func TestOCIManifestList(t *testing.T) { mediaType, canonical, _ := deserialized.Payload() - if mediaType != MediaTypeOCIManifestList { + if mediaType != v1.MediaTypeImageIndex { t.Fatalf("unexpected media type: %s", mediaType) } @@ -187,8 +188,8 @@ func TestOCIManifestList(t *testing.T) { } // Check that the canonical field has the expected value. - if !bytes.Equal(expectedOCIManifestListSerialization, canonical) { - t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedOCIManifestListSerialization)) + if !bytes.Equal(expectedOCIImageIndexSerialization, canonical) { + t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedOCIImageIndexSerialization)) } var unmarshalled DeserializedManifestList diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go index 5318c139..a6d6633a 100644 --- a/manifest/ocischema/builder.go +++ b/manifest/ocischema/builder.go @@ -4,6 +4,7 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) // builder is a type for constructing manifests. @@ -48,7 +49,7 @@ func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { case nil: // Override MediaType, since Put always replaces the specified media // type with application/octet-stream in the descriptor it returns. - m.Config.MediaType = MediaTypeConfig + m.Config.MediaType = v1.MediaTypeImageConfig return FromStruct(m) case distribution.ErrBlobUnknown: // nop @@ -57,10 +58,10 @@ func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { } // Add config to the blob store - m.Config, err = mb.bs.Put(ctx, MediaTypeConfig, mb.configJSON) + m.Config, err = mb.bs.Put(ctx, v1.MediaTypeImageConfig, mb.configJSON) // Override MediaType, since Put always replaces the specified media // type with application/octet-stream in the descriptor it returns. - m.Config.MediaType = MediaTypeConfig + m.Config.MediaType = v1.MediaTypeImageConfig if err != nil { return nil, err } diff --git a/manifest/ocischema/builder_test.go b/manifest/ocischema/builder_test.go index 7192ac59..33e9164f 100644 --- a/manifest/ocischema/builder_test.go +++ b/manifest/ocischema/builder_test.go @@ -7,6 +7,7 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/context" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) type mockBlobService struct { @@ -151,17 +152,17 @@ func TestBuilder(t *testing.T) { { Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"), Size: 5312, - MediaType: MediaTypeLayer, + MediaType: v1.MediaTypeImageLayerGzip, }, { Digest: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"), Size: 235231, - MediaType: MediaTypeLayer, + MediaType: v1.MediaTypeImageLayerGzip, }, { Digest: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"), Size: 639152, - MediaType: MediaTypeLayer, + MediaType: v1.MediaTypeImageLayerGzip, }, } @@ -195,7 +196,7 @@ func TestBuilder(t *testing.T) { if target.Digest != configDigest { t.Fatalf("unexpected digest in target: %s", target.Digest.String()) } - if target.MediaType != MediaTypeConfig { + if target.MediaType != v1.MediaTypeImageConfig { t.Fatalf("unexpected media type in target: %s", target.MediaType) } if target.Size != 3153 { diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index 52f64678..f1a05f0a 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -8,32 +8,15 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest" "github.com/opencontainers/go-digest" -) - -const ( - // MediaTypeManifest specifies the mediaType for the current version. - MediaTypeManifest = "application/vnd.oci.image.manifest.v1+json" - - // MediaTypeConfig specifies the mediaType for the image configuration. - MediaTypeConfig = "application/vnd.oci.image.config.v1+json" - - // MediaTypePluginConfig specifies the mediaType for plugin configuration. - MediaTypePluginConfig = "application/vnd.docker.plugin.v1+json" - - // MediaTypeLayer is the mediaType used for layers referenced by the manifest. - MediaTypeLayer = "application/vnd.oci.image.layer.v1.tar+gzip" - - // MediaTypeForeignLayer is the mediaType used for layers that must be - // downloaded from foreign URLs. - MediaTypeForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" + "github.com/opencontainers/image-spec/specs-go/v1" ) var ( // SchemaVersion provides a pre-initialized version structure for this // packages version of the manifest. SchemaVersion = manifest.Versioned{ - SchemaVersion: 2, // Mike: todo this could confusing cause oci version 1 is closer to docker 2 than 1 - MediaType: MediaTypeManifest, + SchemaVersion: 2, // TODO: (mikebrow/stevvooe) this could be confusing cause oci version 1 is closer to docker 2 than 1 + MediaType: v1.MediaTypeImageManifest, } ) @@ -46,15 +29,15 @@ func init() { } dgst := digest.FromBytes(b) - return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifest}, err + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageManifest}, err } - err := distribution.RegisterManifestSchema(MediaTypeManifest, ocischemaFunc) + err := distribution.RegisterManifestSchema(v1.MediaTypeImageManifest, ocischemaFunc) if err != nil { panic(fmt.Sprintf("Unable to register manifest: %s", err)) } } -// Manifest defines a schema2 manifest. +// Manifest defines a ocischema manifest. type Manifest struct { manifest.Versioned @@ -64,6 +47,9 @@ type Manifest struct { // Layers lists descriptors for the layers referenced by the // configuration. Layers []distribution.Descriptor `json:"layers"` + + // Annotations contains arbitrary metadata for the image manifest. + Annotations map[string]string `json:"annotations,omitempty"` } // References returnes the descriptors of this manifests references. @@ -71,6 +57,7 @@ func (m Manifest) References() []distribution.Descriptor { references := make([]distribution.Descriptor, 0, 1+len(m.Layers)) references = append(references, m.Config) references = append(references, m.Layers...) + // TODO: (mikebrow/stevvooe) should we return annotations as references return references } diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go index 9f439dcd..55dc0306 100644 --- a/manifest/ocischema/manifest_test.go +++ b/manifest/ocischema/manifest_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/docker/distribution" + "github.com/opencontainers/image-spec/specs-go/v1" ) var expectedManifestSerialization = []byte(`{ @@ -32,13 +33,13 @@ func TestManifest(t *testing.T) { Config: distribution.Descriptor{ Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", Size: 985, - MediaType: MediaTypeConfig, + MediaType: v1.MediaTypeImageConfig, }, Layers: []distribution.Descriptor{ { Digest: "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b", Size: 153263, - MediaType: MediaTypeLayer, + MediaType: v1.MediaTypeImageLayerGzip, }, }, } @@ -48,9 +49,9 @@ func TestManifest(t *testing.T) { t.Fatalf("error creating DeserializedManifest: %v", err) } - mediaType, canonical, err := deserialized.Payload() + mediaType, canonical, _ := deserialized.Payload() - if mediaType != MediaTypeManifest { + if mediaType != v1.MediaTypeImageManifest { t.Fatalf("unexpected media type: %s", mediaType) } @@ -82,7 +83,7 @@ func TestManifest(t *testing.T) { if target.Digest != "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" { t.Fatalf("unexpected digest in target: %s", target.Digest.String()) } - if target.MediaType != MediaTypeConfig { + if target.MediaType != v1.MediaTypeImageConfig { t.Fatalf("unexpected media type in target: %s", target.MediaType) } if target.Size != 985 { @@ -102,7 +103,7 @@ func TestManifest(t *testing.T) { if references[1].Digest != "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b" { t.Fatalf("unexpected digest in reference: %s", references[0].Digest.String()) } - if references[1].MediaType != MediaTypeLayer { + if references[1].MediaType != v1.MediaTypeImageLayerGzip { t.Fatalf("unexpected media type in reference: %s", references[0].MediaType) } if references[1].Size != 153263 { diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index af0a8876..07fd6f8c 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -18,6 +18,7 @@ import ( "github.com/docker/distribution/registry/auth" "github.com/gorilla/handlers" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) // These constants determine which architecture and OS to choose from a @@ -76,7 +77,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) supportsSchema2 := false supportsManifestList := false supportsOCISchema := false - supportsOCIManifestList := false + supportsOCIImageIndex := false // this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values // https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202 for _, acceptHeader := range r.Header["Accept"] { @@ -100,15 +101,15 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) if mediaType == manifestlist.MediaTypeManifestList { supportsManifestList = true } - if mediaType == ocischema.MediaTypeManifest { + if mediaType == v1.MediaTypeImageManifest { supportsOCISchema = true } - if mediaType == manifestlist.MediaTypeOCIManifestList { - supportsOCIManifestList = true + if mediaType == v1.MediaTypeImageIndex { + supportsOCIImageIndex = true } } } - supportsOCI := supportsOCISchema || supportsOCIManifestList + supportsOCI := supportsOCISchema || supportsOCIImageIndex var manifest distribution.Manifest if imh.Tag != "" { @@ -151,15 +152,15 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest) manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList) - isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == ocischema.MediaTypeManifest) - isAnOCIManifestList := isManifestList && (manifestList.MediaType == manifestlist.MediaTypeOCIManifestList) + isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == v1.MediaTypeImageManifest) + isAnOCIImageIndex := isManifestList && (manifestList.MediaType == v1.MediaTypeImageIndex) if (isSchema2 && !isAnOCIManifest) && (supportsOCISchema && !supportsSchema2) { fmt.Printf("\n\nmanifest is schema2, but accept header only supports OCISchema \n\n") w.WriteHeader(http.StatusNotFound) return } - if (isManifestList && !isAnOCIManifestList) && (supportsOCIManifestList && !supportsManifestList) { + if (isManifestList && !isAnOCIImageIndex) && (supportsOCIImageIndex && !supportsManifestList) { fmt.Printf("\n\nmanifestlist is not OCI, but accept header only supports an OCI manifestlist\n\n") w.WriteHeader(http.StatusNotFound) return @@ -169,7 +170,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNotFound) return } - if isAnOCIManifestList && (!supportsOCIManifestList && supportsManifestList) { + if isAnOCIImageIndex && (!supportsOCIImageIndex && supportsManifestList) { fmt.Printf("\n\nmanifestlist is OCI, but accept header only supports non-OCI manifestlists\n\n") w.WriteHeader(http.StatusNotFound) return @@ -185,7 +186,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) if err != nil { return } - } else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIManifestList) { + } else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIImageIndex) { // Rewrite manifest in schema1 format ctxu.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) @@ -243,7 +244,7 @@ func (imh *manifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Requ supportsSchema2 := false supportsManifestList := false supportsOCISchema := false - supportsOCIManifestList := false + supportsOCIImageIndex := false // this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values // https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202 @@ -268,15 +269,15 @@ func (imh *manifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Requ if mediaType == manifestlist.MediaTypeManifestList { supportsManifestList = true } - if mediaType == ocischema.MediaTypeManifest { + if mediaType == v1.MediaTypeImageManifest { supportsOCISchema = true } - if mediaType == manifestlist.MediaTypeOCIManifestList { - supportsOCIManifestList = true + if mediaType == v1.MediaTypeImageIndex { + supportsOCIImageIndex = true } } } - supportsOCI := supportsOCISchema || supportsOCIManifestList + supportsOCI := supportsOCISchema || supportsOCIImageIndex ctxu.GetLogger(imh).Debug("GetImageManifest") manifests, err := imh.Repository.Manifests(imh) @@ -325,14 +326,14 @@ func (imh *manifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Requ schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest) manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList) - isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == ocischema.MediaTypeManifest) - isAnOCIManifestList := isManifestList && (manifestList.MediaType == manifestlist.MediaTypeOCIManifestList) + isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == v1.MediaTypeImageManifest) + isAnOCIImageIndex := isManifestList && (manifestList.MediaType == v1.MediaTypeImageIndex) badCombinations := [][]bool{ {isSchema2 && !isAnOCIManifest, supportsOCISchema && !supportsSchema2}, - {isManifestList && !isAnOCIManifestList, supportsOCIManifestList && !supportsManifestList}, + {isManifestList && !isAnOCIImageIndex, supportsOCIImageIndex && !supportsManifestList}, {isAnOCIManifest, !supportsOCISchema && supportsSchema2}, - {isAnOCIManifestList, !supportsOCIManifestList && supportsManifestList}, + {isAnOCIImageIndex, !supportsOCIImageIndex && supportsManifestList}, } for i, combo := range badCombinations { if combo[0] && combo[1] { @@ -360,7 +361,7 @@ func (imh *manifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Requ if err != nil { return } - } else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIManifestList) { + } else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIImageIndex) { // Rewrite manifest in schema1 format dcontext.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) @@ -499,7 +500,7 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) return } - isAnOCIManifest := mediaType == ocischema.MediaTypeManifest || mediaType == manifestlist.MediaTypeOCIManifestList + isAnOCIManifest := mediaType == v1.MediaTypeImageManifest || mediaType == v1.MediaTypeImageIndex if isAnOCIManifest { fmt.Printf("\n\nPutting an OCI Manifest!\n\n\n") @@ -615,6 +616,14 @@ func (imh *manifestHandler) applyResourcePolicy(manifest distribution.Manifest) message := fmt.Sprintf("unknown manifest class for %s", m.Config.MediaType) return errcode.ErrorCodeDenied.WithMessage(message) } + case *ocischema.DeserializedManifest: + switch m.Config.MediaType { + case v1.MediaTypeImageConfig: + class = "image" + default: + message := fmt.Sprintf("unknown manifest class for %s", m.Config.MediaType) + return errcode.ErrorCodeDenied.WithMessage(message) + } } if class == "" { diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 13b4f5b5..6543f6fd 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -13,6 +13,7 @@ import ( "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) // A ManifestHandler gets and puts manifests of a particular type. @@ -101,9 +102,9 @@ func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options .. switch versioned.MediaType { case schema2.MediaTypeManifest: return ms.schema2Handler.Unmarshal(ctx, dgst, content) - case ocischema.MediaTypeManifest: + case v1.MediaTypeImageManifest: return ms.ocischemaHandler.Unmarshal(ctx, dgst, content) - case manifestlist.MediaTypeManifestList, manifestlist.MediaTypeOCIManifestList: + case manifestlist.MediaTypeManifestList, v1.MediaTypeImageIndex: return ms.manifestListHandler.Unmarshal(ctx, dgst, content) default: return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)} diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index bb658348..148b7a2e 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -9,6 +9,7 @@ import ( "github.com/docker/distribution/context" "github.com/docker/distribution/manifest/ocischema" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) //ocischemaManifestHandler is a ManifestHandler that covers ocischema manifests. @@ -79,7 +80,8 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc var err error switch descriptor.MediaType { - case ocischema.MediaTypeForeignLayer: + // TODO: mikebrow/steveoe verify we should treat oci nondistributable like foreign layers? + case v1.MediaTypeImageLayerNonDistributable, v1.MediaTypeImageLayerNonDistributableGzip: // Clients download this layer from an external URL, so do not check for // its presense. if len(descriptor.URLs) == 0 { @@ -95,7 +97,7 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc break } } - case ocischema.MediaTypeManifest: + case v1.MediaTypeImageManifest: var exists bool exists, err = manifestService.Exists(ctx, descriptor.Digest) if err != nil || !exists { diff --git a/registry/storage/ocimanifesthandler_test.go b/registry/storage/ocimanifesthandler_test.go index e09f1ee3..d04ce531 100644 --- a/registry/storage/ocimanifesthandler_test.go +++ b/registry/storage/ocimanifesthandler_test.go @@ -9,9 +9,10 @@ import ( "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/registry/storage/driver/inmemory" + "github.com/opencontainers/image-spec/specs-go/v1" ) -func TestVerifyOCIManifestForeignLayer(t *testing.T) { +func TestVerifyOCIManifestNonDistributableLayer(t *testing.T) { ctx := context.Background() inmemoryDriver := inmemory.New() registry := createRegistry(t, inmemoryDriver, @@ -20,26 +21,26 @@ func TestVerifyOCIManifestForeignLayer(t *testing.T) { repo := makeRepository(t, registry, "test") manifestService := makeManifestService(t, repo) - config, err := repo.Blobs(ctx).Put(ctx, ocischema.MediaTypeConfig, nil) + config, err := repo.Blobs(ctx).Put(ctx, v1.MediaTypeImageConfig, nil) if err != nil { t.Fatal(err) } - layer, err := repo.Blobs(ctx).Put(ctx, ocischema.MediaTypeLayer, nil) + layer, err := repo.Blobs(ctx).Put(ctx, v1.MediaTypeImageLayerGzip, nil) if err != nil { t.Fatal(err) } - foreignLayer := distribution.Descriptor{ + nonDistributableLayer := distribution.Descriptor{ Digest: "sha256:463435349086340864309863409683460843608348608934092322395278926a", Size: 6323, - MediaType: ocischema.MediaTypeForeignLayer, + MediaType: v1.MediaTypeImageLayerNonDistributableGzip, } template := ocischema.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 2, - MediaType: ocischema.MediaTypeManifest, + MediaType: v1.MediaTypeImageManifest, }, Config: config, } @@ -52,58 +53,58 @@ func TestVerifyOCIManifestForeignLayer(t *testing.T) { cases := []testcase{ { - foreignLayer, + nonDistributableLayer, nil, errMissingURL, }, { - // regular layers may have foreign urls + // regular layers may have foreign urls (non-Distributable Layers) layer, []string{"http://foo/bar"}, nil, }, { - foreignLayer, + nonDistributableLayer, []string{"file:///local/file"}, errInvalidURL, }, { - foreignLayer, + nonDistributableLayer, []string{"http://foo/bar#baz"}, errInvalidURL, }, { - foreignLayer, + nonDistributableLayer, []string{""}, errInvalidURL, }, { - foreignLayer, + nonDistributableLayer, []string{"https://foo/bar", ""}, errInvalidURL, }, { - foreignLayer, + nonDistributableLayer, []string{"", "https://foo/bar"}, errInvalidURL, }, { - foreignLayer, + nonDistributableLayer, []string{"http://nope/bar"}, errInvalidURL, }, { - foreignLayer, + nonDistributableLayer, []string{"http://foo/nope"}, errInvalidURL, }, { - foreignLayer, + nonDistributableLayer, []string{"http://foo/bar"}, nil, }, { - foreignLayer, + nonDistributableLayer, []string{"https://foo/bar"}, nil, }, From fcaffa38bcbb6055a626b22bb5c6c6488c92833e Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Tue, 11 Jul 2017 14:20:34 -0500 Subject: [PATCH 04/20] vendor oci image-spec Signed-off-by: Mike Brown --- vendor.conf | 1 + .../opencontainers/image-spec/LICENSE | 191 ++++++++++++++++++ .../opencontainers/image-spec/README.md | 167 +++++++++++++++ .../image-spec/specs-go/v1/annotations.go | 56 +++++ .../image-spec/specs-go/v1/config.go | 103 ++++++++++ .../image-spec/specs-go/v1/descriptor.go | 64 ++++++ .../image-spec/specs-go/v1/index.go | 29 +++ .../image-spec/specs-go/v1/layout.go | 28 +++ .../image-spec/specs-go/v1/manifest.go | 32 +++ .../image-spec/specs-go/v1/mediatype.go | 48 +++++ .../image-spec/specs-go/version.go | 32 +++ .../image-spec/specs-go/versioned.go | 23 +++ 12 files changed, 774 insertions(+) create mode 100644 vendor/github.com/opencontainers/image-spec/LICENSE create mode 100644 vendor/github.com/opencontainers/image-spec/README.md create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/config.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/descriptor.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/index.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/layout.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/manifest.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/v1/mediatype.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/version.go create mode 100644 vendor/github.com/opencontainers/image-spec/specs-go/versioned.go diff --git a/vendor.conf b/vendor.conf index b67e03ad..4f52b597 100644 --- a/vendor.conf +++ b/vendor.conf @@ -49,3 +49,4 @@ gopkg.in/square/go-jose.v1 40d457b439244b546f023d056628e5184136899b gopkg.in/yaml.v2 bef53efd0c76e49e6de55ead051f886bea7e9420 rsc.io/letsencrypt e770c10b0f1a64775ae91d240407ce00d1a5bdeb https://github.com/dmcgowan/letsencrypt.git github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb +github.com/opencontainers/image-spec e0536ae0cb47b8fed6afdaa9ba2589f6bd915caa diff --git a/vendor/github.com/opencontainers/image-spec/LICENSE b/vendor/github.com/opencontainers/image-spec/LICENSE new file mode 100644 index 00000000..9fdc20fd --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/LICENSE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2016 The Linux Foundation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/opencontainers/image-spec/README.md b/vendor/github.com/opencontainers/image-spec/README.md new file mode 100644 index 00000000..5ab5554e --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/README.md @@ -0,0 +1,167 @@ +# OCI Image Format Specification +
+ + + +
+ +The OCI Image Format project creates and maintains the software shipping container image format spec (OCI Image Format). + +**[The specification can be found here](spec.md).** + +This repository also provides [Go types](specs-go), [intra-blob validation tooling, and JSON Schema](schema). +The Go types and validation should be compatible with the current Go release; earlier Go releases are not supported. + +Additional documentation about how this group operates: + +- [Code of Conduct](https://github.com/opencontainers/tob/blob/d2f9d68c1332870e40693fe077d311e0742bc73d/code-of-conduct.md) +- [Roadmap](#roadmap) +- [Releases](RELEASES.md) +- [Project Documentation](project.md) + +The _optional_ and _base_ layers of all OCI projects are tracked in the [OCI Scope Table](https://www.opencontainers.org/about/oci-scope-table). + +## Running an OCI Image + +The OCI Image Format partner project is the [OCI Runtime Spec project](https://github.com/opencontainers/runtime-spec). +The Runtime Specification outlines how to run a "[filesystem bundle](https://github.com/opencontainers/runtime-spec/blob/master/bundle.md)" that is unpacked on disk. +At a high-level an OCI implementation would download an OCI Image then unpack that image into an OCI Runtime filesystem bundle. +At this point the OCI Runtime Bundle would be run by an OCI Runtime. + +This entire workflow supports the UX that users have come to expect from container engines like Docker and rkt: primarily, the ability to run an image with no additional arguments: + +* docker run example.com/org/app:v1.0.0 +* rkt run example.com/org/app,version=v1.0.0 + +To support this UX the OCI Image Format contains sufficient information to launch the application on the target platform (e.g. command, arguments, environment variables, etc). + +## FAQ + +**Q: Why doesn't this project mention distribution?** + +A: Distribution, for example using HTTP as both Docker v2.2 and AppC do today, is currently out of scope on the [OCI Scope Table](https://www.opencontainers.org/about/oci-scope-table). +There has been [some discussion on the TOB mailing list](https://groups.google.com/a/opencontainers.org/d/msg/tob/A3JnmI-D-6Y/tLuptPDHAgAJ) to make distribution an optional layer, but this topic is a work in progress. + +**Q: What happens to AppC or Docker Image Formats?** + +A: Existing formats can continue to be a proving ground for technologies, as needed. +The OCI Image Format project strives to provide a dependable open specification that can be shared between different tools and be evolved for years or decades of compatibility; as the deb and rpm format have. + +Find more [FAQ on the OCI site](https://www.opencontainers.org/faq). + +## Roadmap + +The [GitHub milestones](https://github.com/opencontainers/image-spec/milestones) lay out the path to the OCI v1.0.0 release in late 2016. + +# Contributing + +Development happens on GitHub for the spec. +Issues are used for bugs and actionable items and longer discussions can happen on the [mailing list](#mailing-list). + +The specification and code is licensed under the Apache 2.0 license found in the `LICENSE` file of this repository. + +## Discuss your design + +The project welcomes submissions, but please let everyone know what you are working on. + +Before undertaking a nontrivial change to this specification, send mail to the [mailing list](#mailing-list) to discuss what you plan to do. +This gives everyone a chance to validate the design, helps prevent duplication of effort, and ensures that the idea fits. +It also guarantees that the design is sound before code is written; a GitHub pull-request is not the place for high-level discussions. + +Typos and grammatical errors can go straight to a pull-request. +When in doubt, start on the [mailing-list](#mailing-list). + +## Weekly Call + +The contributors and maintainers of all OCI projects have a weekly meeting Wednesdays at 2:00 PM (USA Pacific). +Everyone is welcome to participate via [UberConference web][UberConference] or audio-only: +1-415-968-0849 (no PIN needed). +An initial agenda will be posted to the [mailing list](#mailing-list) earlier in the week, and everyone is welcome to propose additional topics or suggest other agenda alterations there. +Minutes are posted to the [mailing list](#mailing-list) and minutes from past calls are archived [here][minutes]. + +## Mailing List + +You can subscribe and join the mailing list on [Google Groups](https://groups.google.com/a/opencontainers.org/forum/#!forum/dev). + +## IRC + +OCI discussion happens on #opencontainers on Freenode ([logs][irc-logs]). + +## Markdown style + +To keep consistency throughout the Markdown files in the Open Container spec all files should be formatted one sentence per line. +This fixes two things: it makes diffing easier with git and it resolves fights about line wrapping length. +For example, this paragraph will span three lines in the Markdown source. + +## Git commit + +### Sign your work + +The sign-off is a simple line at the end of the explanation for the patch, which certifies that you wrote it or otherwise have the right to pass it on as an open-source patch. +The rules are pretty simple: if you can certify the below (from [developercertificate.org](http://developercertificate.org/)): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +660 York Street, Suite 102, +San Francisco, CA 94110 USA + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` + +then you just add a line to every git commit message: + + Signed-off-by: Joe Smith + +using your real name (sorry, no pseudonyms or anonymous contributions.) + +You can add the sign off when creating the git commit via `git commit -s`. + +### Commit Style + +Simple house-keeping for clean git history. +Read more on [How to Write a Git Commit Message](http://chris.beams.io/posts/git-commit/) or the Discussion section of [`git-commit(1)`](http://git-scm.com/docs/git-commit). + +1. Separate the subject from body with a blank line +2. Limit the subject line to 50 characters +3. Capitalize the subject line +4. Do not end the subject line with a period +5. Use the imperative mood in the subject line +6. Wrap the body at 72 characters +7. Use the body to explain what and why vs. how + * If there was important/useful/essential conversation or information, copy or include a reference +8. When possible, one keyword to scope the change in the subject (i.e. "README: ...", "runtime: ...") + + +[UberConference]: https://www.uberconference.com/opencontainers +[irc-logs]: http://ircbot.wl.linuxfoundation.org/eavesdrop/%23opencontainers/ +[minutes]: http://ircbot.wl.linuxfoundation.org/meetings/opencontainers/ diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go new file mode 100644 index 00000000..35d81089 --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/annotations.go @@ -0,0 +1,56 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +const ( + // AnnotationCreated is the annotation key for the date and time on which the image was built (date-time string as defined by RFC 3339). + AnnotationCreated = "org.opencontainers.image.created" + + // AnnotationAuthors is the annotation key for the contact details of the people or organization responsible for the image (freeform string). + AnnotationAuthors = "org.opencontainers.image.authors" + + // AnnotationURL is the annotation key for the URL to find more information on the image. + AnnotationURL = "org.opencontainers.image.url" + + // AnnotationDocumentation is the annotation key for the URL to get documentation on the image. + AnnotationDocumentation = "org.opencontainers.image.documentation" + + // AnnotationSource is the annotation key for the URL to get source code for building the image. + AnnotationSource = "org.opencontainers.image.source" + + // AnnotationVersion is the annotation key for the version of the packaged software. + // The version MAY match a label or tag in the source code repository. + // The version MAY be Semantic versioning-compatible. + AnnotationVersion = "org.opencontainers.image.version" + + // AnnotationRevision is the annotation key for the source control revision identifier for the packaged software. + AnnotationRevision = "org.opencontainers.image.revision" + + // AnnotationVendor is the annotation key for the name of the distributing entity, organization or individual. + AnnotationVendor = "org.opencontainers.image.vendor" + + // AnnotationLicenses is the annotation key for the license(s) under which contained software is distributed as an SPDX License Expression. + AnnotationLicenses = "org.opencontainers.image.licenses" + + // AnnotationRefName is the annotation key for the name of the reference for a target. + // SHOULD only be considered valid when on descriptors on `index.json` within image layout. + AnnotationRefName = "org.opencontainers.image.ref.name" + + // AnnotationTitle is the annotation key for the human-readable title of the image. + AnnotationTitle = "org.opencontainers.image.title" + + // AnnotationDescription is the annotation key for the human-readable description of the software packaged in the image. + AnnotationDescription = "org.opencontainers.image.description" +) diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/config.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/config.go new file mode 100644 index 00000000..fe799bd6 --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/config.go @@ -0,0 +1,103 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "time" + + digest "github.com/opencontainers/go-digest" +) + +// ImageConfig defines the execution parameters which should be used as a base when running a container using an image. +type ImageConfig struct { + // User defines the username or UID which the process in the container should run as. + User string `json:"User,omitempty"` + + // ExposedPorts a set of ports to expose from a container running this image. + ExposedPorts map[string]struct{} `json:"ExposedPorts,omitempty"` + + // Env is a list of environment variables to be used in a container. + Env []string `json:"Env,omitempty"` + + // Entrypoint defines a list of arguments to use as the command to execute when the container starts. + Entrypoint []string `json:"Entrypoint,omitempty"` + + // Cmd defines the default arguments to the entrypoint of the container. + Cmd []string `json:"Cmd,omitempty"` + + // Volumes is a set of directories describing where the process is likely write data specific to a container instance. + Volumes map[string]struct{} `json:"Volumes,omitempty"` + + // WorkingDir sets the current working directory of the entrypoint process in the container. + WorkingDir string `json:"WorkingDir,omitempty"` + + // Labels contains arbitrary metadata for the container. + Labels map[string]string `json:"Labels,omitempty"` + + // StopSignal contains the system call signal that will be sent to the container to exit. + StopSignal string `json:"StopSignal,omitempty"` +} + +// RootFS describes a layer content addresses +type RootFS struct { + // Type is the type of the rootfs. + Type string `json:"type"` + + // DiffIDs is an array of layer content hashes (DiffIDs), in order from bottom-most to top-most. + DiffIDs []digest.Digest `json:"diff_ids"` +} + +// History describes the history of a layer. +type History struct { + // Created is the combined date and time at which the layer was created, formatted as defined by RFC 3339, section 5.6. + Created *time.Time `json:"created,omitempty"` + + // CreatedBy is the command which created the layer. + CreatedBy string `json:"created_by,omitempty"` + + // Author is the author of the build point. + Author string `json:"author,omitempty"` + + // Comment is a custom message set when creating the layer. + Comment string `json:"comment,omitempty"` + + // EmptyLayer is used to mark if the history item created a filesystem diff. + EmptyLayer bool `json:"empty_layer,omitempty"` +} + +// Image is the JSON structure which describes some basic information about the image. +// This provides the `application/vnd.oci.image.config.v1+json` mediatype when marshalled to JSON. +type Image struct { + // Created is the combined date and time at which the image was created, formatted as defined by RFC 3339, section 5.6. + Created *time.Time `json:"created,omitempty"` + + // Author defines the name and/or email address of the person or entity which created and is responsible for maintaining the image. + Author string `json:"author,omitempty"` + + // Architecture is the CPU architecture which the binaries in this image are built to run on. + Architecture string `json:"architecture"` + + // OS is the name of the operating system which the image is built to run on. + OS string `json:"os"` + + // Config defines the execution parameters which should be used as a base when running a container using the image. + Config ImageConfig `json:"config,omitempty"` + + // RootFS references the layer content addresses used by the image. + RootFS RootFS `json:"rootfs"` + + // History describes the history of each layer. + History []History `json:"history,omitempty"` +} diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/descriptor.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/descriptor.go new file mode 100644 index 00000000..6e442a08 --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/descriptor.go @@ -0,0 +1,64 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import digest "github.com/opencontainers/go-digest" + +// Descriptor describes the disposition of targeted content. +// This structure provides `application/vnd.oci.descriptor.v1+json` mediatype +// when marshalled to JSON. +type Descriptor struct { + // MediaType is the media type of the object this schema refers to. + MediaType string `json:"mediaType,omitempty"` + + // Digest is the digest of the targeted content. + Digest digest.Digest `json:"digest"` + + // Size specifies the size in bytes of the blob. + Size int64 `json:"size"` + + // URLs specifies a list of URLs from which this object MAY be downloaded + URLs []string `json:"urls,omitempty"` + + // Annotations contains arbitrary metadata relating to the targeted content. + Annotations map[string]string `json:"annotations,omitempty"` + + // Platform describes the platform which the image in the manifest runs on. + // + // This should only be used when referring to a manifest. + Platform *Platform `json:"platform,omitempty"` +} + +// Platform describes the platform which the image in the manifest runs on. +type Platform struct { + // Architecture field specifies the CPU architecture, for example + // `amd64` or `ppc64`. + Architecture string `json:"architecture"` + + // OS specifies the operating system, for example `linux` or `windows`. + OS string `json:"os"` + + // OSVersion is an optional field specifying the operating system + // version, for example on Windows `10.0.14393.1066`. + OSVersion string `json:"os.version,omitempty"` + + // OSFeatures is an optional field specifying an array of strings, + // each listing a required OS feature (for example on Windows `win32k`). + OSFeatures []string `json:"os.features,omitempty"` + + // Variant is an optional field specifying a variant of the CPU, for + // example `v7` to specify ARMv7 when architecture is `arm`. + Variant string `json:"variant,omitempty"` +} diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/index.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/index.go new file mode 100644 index 00000000..4e6c4b23 --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/index.go @@ -0,0 +1,29 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import "github.com/opencontainers/image-spec/specs-go" + +// Index references manifests for various platforms. +// This structure provides `application/vnd.oci.image.index.v1+json` mediatype when marshalled to JSON. +type Index struct { + specs.Versioned + + // Manifests references platform specific manifests. + Manifests []Descriptor `json:"manifests"` + + // Annotations contains arbitrary metadata for the image index. + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/layout.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/layout.go new file mode 100644 index 00000000..fc79e9e0 --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/layout.go @@ -0,0 +1,28 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +const ( + // ImageLayoutFile is the file name of oci image layout file + ImageLayoutFile = "oci-layout" + // ImageLayoutVersion is the version of ImageLayout + ImageLayoutVersion = "1.0.0" +) + +// ImageLayout is the structure in the "oci-layout" file, found in the root +// of an OCI Image-layout directory. +type ImageLayout struct { + Version string `json:"imageLayoutVersion"` +} diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/manifest.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/manifest.go new file mode 100644 index 00000000..7ff32c40 --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/manifest.go @@ -0,0 +1,32 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import "github.com/opencontainers/image-spec/specs-go" + +// Manifest provides `application/vnd.oci.image.manifest.v1+json` mediatype structure when marshalled to JSON. +type Manifest struct { + specs.Versioned + + // Config references a configuration object for a container, by digest. + // The referenced configuration object is a JSON blob that the runtime uses to set up the container. + Config Descriptor `json:"config"` + + // Layers is an indexed list of layers referenced by the manifest. + Layers []Descriptor `json:"layers"` + + // Annotations contains arbitrary metadata for the image manifest. + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/v1/mediatype.go b/vendor/github.com/opencontainers/image-spec/specs-go/v1/mediatype.go new file mode 100644 index 00000000..bad7bb97 --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/v1/mediatype.go @@ -0,0 +1,48 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +const ( + // MediaTypeDescriptor specifies the media type for a content descriptor. + MediaTypeDescriptor = "application/vnd.oci.descriptor.v1+json" + + // MediaTypeLayoutHeader specifies the media type for the oci-layout. + MediaTypeLayoutHeader = "application/vnd.oci.layout.header.v1+json" + + // MediaTypeImageManifest specifies the media type for an image manifest. + MediaTypeImageManifest = "application/vnd.oci.image.manifest.v1+json" + + // MediaTypeImageIndex specifies the media type for an image index. + MediaTypeImageIndex = "application/vnd.oci.image.index.v1+json" + + // MediaTypeImageLayer is the media type used for layers referenced by the manifest. + MediaTypeImageLayer = "application/vnd.oci.image.layer.v1.tar" + + // MediaTypeImageLayerGzip is the media type used for gzipped layers + // referenced by the manifest. + MediaTypeImageLayerGzip = "application/vnd.oci.image.layer.v1.tar+gzip" + + // MediaTypeImageLayerNonDistributable is the media type for layers referenced by + // the manifest but with distribution restrictions. + MediaTypeImageLayerNonDistributable = "application/vnd.oci.image.layer.nondistributable.v1.tar" + + // MediaTypeImageLayerNonDistributableGzip is the media type for + // gzipped layers referenced by the manifest but with distribution + // restrictions. + MediaTypeImageLayerNonDistributableGzip = "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip" + + // MediaTypeImageConfig specifies the media type for the image configuration. + MediaTypeImageConfig = "application/vnd.oci.image.config.v1+json" +) diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/version.go b/vendor/github.com/opencontainers/image-spec/specs-go/version.go new file mode 100644 index 00000000..f4cda6ed --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/version.go @@ -0,0 +1,32 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package specs + +import "fmt" + +const ( + // VersionMajor is for an API incompatible changes + VersionMajor = 1 + // VersionMinor is for functionality in a backwards-compatible manner + VersionMinor = 0 + // VersionPatch is for backwards-compatible bug fixes + VersionPatch = 0 + + // VersionDev indicates development branch. Releases will be empty string. + VersionDev = "-rc6-dev" +) + +// Version is the specification version that the package types support. +var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev) diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/versioned.go b/vendor/github.com/opencontainers/image-spec/specs-go/versioned.go new file mode 100644 index 00000000..58a1510f --- /dev/null +++ b/vendor/github.com/opencontainers/image-spec/specs-go/versioned.go @@ -0,0 +1,23 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package specs + +// Versioned provides a struct with the manifest schemaVersion and mediaType. +// Incoming content with unknown schema version can be decoded against this +// struct to check the version. +type Versioned struct { + // SchemaVersion is the image manifest schema that this image follows + SchemaVersion int `json:"schemaVersion"` +} From 426afb3a4c17a619273cfba8ad169b10418f5627 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Wed, 12 Jul 2017 18:07:46 -0500 Subject: [PATCH 05/20] address get manifest issue with oci. namespace; and comment descriptions Signed-off-by: Mike Brown --- manifest/ocischema/manifest.go | 2 +- manifest/schema2/manifest.go | 2 +- registry/handlers/manifests.go | 188 ++------------------------------- 3 files changed, 9 insertions(+), 183 deletions(-) diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index f1a05f0a..a5b199d4 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -61,7 +61,7 @@ func (m Manifest) References() []distribution.Descriptor { return references } -// Target returns the target of this signed manifest. +// Target returns the target of this manifest. func (m Manifest) Target() distribution.Descriptor { return m.Config } diff --git a/manifest/schema2/manifest.go b/manifest/schema2/manifest.go index a2708c75..26a6a334 100644 --- a/manifest/schema2/manifest.go +++ b/manifest/schema2/manifest.go @@ -79,7 +79,7 @@ func (m Manifest) References() []distribution.Descriptor { return references } -// Target returns the target of this signed manifest. +// Target returns the target of this manifest. func (m Manifest) Target() distribution.Descriptor { return m.Config } diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index 07fd6f8c..c6e17ada 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -111,13 +111,11 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) } supportsOCI := supportsOCISchema || supportsOCIImageIndex - var manifest distribution.Manifest if imh.Tag != "" { tags := imh.Repository.Tags(imh) var desc distribution.Descriptor - if !supportsOCI { - desc, err = tags.Get(imh, imh.Tag) - } else { + desc, err = tags.Get(imh, imh.annotatedTag(supportsOCI)) + if err != nil && supportsOCI { // in the supportsOCI case fall back to the non OCI image desc, err = tags.Get(imh, imh.annotatedTag(false)) } if err != nil { @@ -140,7 +138,12 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) if imh.Tag != "" { options = append(options, distribution.WithTag(imh.annotatedTag(supportsOCI))) } + var manifest distribution.Manifest manifest, err = manifests.Get(imh, imh.Digest, options...) + if err != nil && supportsOCI && imh.Tag != "" { // in the supportsOCI case fall back to the non OCI image + options = append(options[:0], distribution.WithTag(imh.annotatedTag(false))) + manifest, err = manifests.Get(imh, imh.Digest, options...) + } if err != nil { if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) @@ -238,183 +241,6 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) w.Write(p) } -// GetImageManifest fetches the image manifest from the storage backend, if it exists. -func (imh *manifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) { - fmt.Printf("\n\nGetting a manifest!\n\n\n") - supportsSchema2 := false - supportsManifestList := false - supportsOCISchema := false - supportsOCIImageIndex := false - - // this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values - // https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202 - for _, acceptHeader := range r.Header["Accept"] { - // r.Header[...] is a slice in case the request contains the same header more than once - // if the header isn't set, we'll get the zero value, which "range" will handle gracefully - - // we need to split each header value on "," to get the full list of "Accept" values (per RFC 2616) - // https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.1 - for _, mediaType := range strings.Split(acceptHeader, ",") { - // remove "; q=..." if present - if i := strings.Index(mediaType, ";"); i >= 0 { - mediaType = mediaType[:i] - } - - // it's common (but not required) for Accept values to be space separated ("a/b, c/d, e/f") - mediaType = strings.TrimSpace(mediaType) - - if mediaType == schema2.MediaTypeManifest { - supportsSchema2 = true - } - if mediaType == manifestlist.MediaTypeManifestList { - supportsManifestList = true - } - if mediaType == v1.MediaTypeImageManifest { - supportsOCISchema = true - } - if mediaType == v1.MediaTypeImageIndex { - supportsOCIImageIndex = true - } - } - } - supportsOCI := supportsOCISchema || supportsOCIImageIndex - - ctxu.GetLogger(imh).Debug("GetImageManifest") - manifests, err := imh.Repository.Manifests(imh) - if err != nil { - imh.Errors = append(imh.Errors, err) - return - } - - var manifest distribution.Manifest - if imh.Tag != "" { - tags := imh.Repository.Tags(imh) - var desc distribution.Descriptor - if !supportsOCI { - desc, err = tags.Get(imh, imh.Tag) - if err != nil { - imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) - return - } - } else { - desc, err = tags.Get(imh, imh.annotatedTag(supportsOCI)) - if err != nil { - desc, err = tags.Get(imh, imh.annotatedTag(false)) - if err != nil { - imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) - return - } - } - } - imh.Digest = desc.Digest - } - - if etagMatch(r, imh.Digest.String()) { - w.WriteHeader(http.StatusNotModified) - return - } - - var options []distribution.ManifestServiceOption - if imh.Tag != "" { - options = append(options, distribution.WithTag(imh.annotatedTag(supportsOCI))) - } - manifest, err = manifests.Get(imh, imh.Digest, options...) - if err != nil { - imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) - return - } - - schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest) - manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList) - isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == v1.MediaTypeImageManifest) - isAnOCIImageIndex := isManifestList && (manifestList.MediaType == v1.MediaTypeImageIndex) - - badCombinations := [][]bool{ - {isSchema2 && !isAnOCIManifest, supportsOCISchema && !supportsSchema2}, - {isManifestList && !isAnOCIImageIndex, supportsOCIImageIndex && !supportsManifestList}, - {isAnOCIManifest, !supportsOCISchema && supportsSchema2}, - {isAnOCIImageIndex, !supportsOCIImageIndex && supportsManifestList}, - } - for i, combo := range badCombinations { - if combo[0] && combo[1] { - fmt.Printf("\n\nbad combo! %d\n\n\n", i) - w.WriteHeader(http.StatusNotFound) - return - } - } - if isAnOCIManifest { - fmt.Print("\n\nreturning OCI manifest\n\n") - } else if isSchema2 { - fmt.Print("\n\nreturning schema 2 manifest\n\n") - } else { - fmt.Print("\n\nreturning schema 1 manifest\n\n") - } - - // Only rewrite schema2 manifests when they are being fetched by tag. - // If they are being fetched by digest, we can't return something not - // matching the digest. - if imh.Tag != "" && isSchema2 && !(supportsSchema2 || supportsOCISchema) { - // Rewrite manifest in schema1 format - dcontext.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String()) - - manifest, err = imh.convertSchema2Manifest(schema2Manifest) - if err != nil { - return - } - } else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIImageIndex) { - // Rewrite manifest in schema1 format - dcontext.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) - - // Find the image manifest corresponding to the default - // platform - var manifestDigest digest.Digest - for _, manifestDescriptor := range manifestList.Manifests { - if manifestDescriptor.Platform.Architecture == defaultArch && manifestDescriptor.Platform.OS == defaultOS { - manifestDigest = manifestDescriptor.Digest - break - } - } - - if manifestDigest == "" { - imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown) - return - } - - manifest, err = manifests.Get(imh, manifestDigest) - if err != nil { - if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { - imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) - } else { - imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) - } - return - } - - // If necessary, convert the image manifest - if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !supportsSchema2 { - manifest, err = imh.convertSchema2Manifest(schema2Manifest) - if err != nil { - return - } - } else { - imh.Digest = manifestDigest - } - } - - ct, p, err := manifest.Payload() - if err != nil { - return - } - - w.Header().Set("Content-Type", ct) - w.Header().Set("Content-Length", fmt.Sprint(len(p))) - w.Header().Set("Docker-Content-Digest", imh.Digest.String()) - w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest)) - w.Write(p) - - fmt.Printf("\n\nSucceeded in getting the manifest!\n\n\n") -} - func (imh *manifestHandler) convertSchema2Manifest(schema2Manifest *schema2.DeserializedManifest) (distribution.Manifest, error) { targetDescriptor := schema2Manifest.Target() blobs := imh.Repository.Blobs(imh) From b0cef05626043df250d2ddc6ab84c87b6913b879 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Tue, 18 Jul 2017 16:20:37 -0500 Subject: [PATCH 06/20] removes oci. namespace feature Signed-off-by: Mike Brown --- registry/handlers/manifests.go | 38 ++++++++++------------------------ 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index c6e17ada..ae52bff9 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -27,6 +27,7 @@ const ( defaultArch = "amd64" defaultOS = "linux" maxManifestBodySize = 4 << 20 + imageClass = "image" ) // manifestDispatcher takes the request context and builds the @@ -109,15 +110,11 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) } } } - supportsOCI := supportsOCISchema || supportsOCIImageIndex if imh.Tag != "" { tags := imh.Repository.Tags(imh) var desc distribution.Descriptor - desc, err = tags.Get(imh, imh.annotatedTag(supportsOCI)) - if err != nil && supportsOCI { // in the supportsOCI case fall back to the non OCI image - desc, err = tags.Get(imh, imh.annotatedTag(false)) - } + desc, err = tags.Get(imh, imh.Tag) if err != nil { if _, ok := err.(distribution.ErrTagUnknown); ok { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) @@ -136,14 +133,10 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) var options []distribution.ManifestServiceOption if imh.Tag != "" { - options = append(options, distribution.WithTag(imh.annotatedTag(supportsOCI))) + options = append(options, distribution.WithTag(imh.Tag)) } var manifest distribution.Manifest manifest, err = manifests.Get(imh, imh.Digest, options...) - if err != nil && supportsOCI && imh.Tag != "" { // in the supportsOCI case fall back to the non OCI image - options = append(options[:0], distribution.WithTag(imh.annotatedTag(false))) - manifest, err = manifests.Get(imh, imh.Digest, options...) - } if err != nil { if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) @@ -266,7 +259,7 @@ func (imh *manifestHandler) convertSchema2Manifest(schema2Manifest *schema2.Dese builder := schema1.NewConfigManifestBuilder(imh.Repository.Blobs(imh), imh.Context.App.trustKey, ref, configJSON) for _, d := range schema2Manifest.Layers { - if err := builder.AppendReference(d); err != nil { + if err = builder.AppendReference(d); err != nil { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err)) return nil, err } @@ -336,10 +329,10 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) var options []distribution.ManifestServiceOption if imh.Tag != "" { - options = append(options, distribution.WithTag(imh.annotatedTag(isAnOCIManifest))) + options = append(options, distribution.WithTag(imh.Tag)) } - if err := imh.applyResourcePolicy(manifest); err != nil { + if err = imh.applyResourcePolicy(manifest); err != nil { imh.Errors = append(imh.Errors, err) return } @@ -388,7 +381,7 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) // Tag this manifest if imh.Tag != "" { tags := imh.Repository.Tags(imh) - err = tags.Tag(imh, imh.annotatedTag(isAnOCIManifest), desc) + err = tags.Tag(imh, imh.Tag, desc) if err != nil { fmt.Printf("\n\nXXX 4: %T: %v\n\n\n", err, err) imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) @@ -431,11 +424,11 @@ func (imh *manifestHandler) applyResourcePolicy(manifest distribution.Manifest) var class string switch m := manifest.(type) { case *schema1.SignedManifest: - class = "image" + class = imageClass case *schema2.DeserializedManifest: switch m.Config.MediaType { case schema2.MediaTypeImageConfig: - class = "image" + class = imageClass case schema2.MediaTypePluginConfig: class = "plugin" default: @@ -445,7 +438,7 @@ func (imh *manifestHandler) applyResourcePolicy(manifest distribution.Manifest) case *ocischema.DeserializedManifest: switch m.Config.MediaType { case v1.MediaTypeImageConfig: - class = "image" + class = imageClass default: message := fmt.Sprintf("unknown manifest class for %s", m.Config.MediaType) return errcode.ErrorCodeDenied.WithMessage(message) @@ -476,7 +469,7 @@ func (imh *manifestHandler) applyResourcePolicy(manifest distribution.Manifest) for _, r := range resources { if r.Name == n { if r.Class == "" { - r.Class = "image" + r.Class = imageClass } if r.Class == class { return nil @@ -540,12 +533,3 @@ func (imh *manifestHandler) DeleteManifest(w http.ResponseWriter, r *http.Reques w.WriteHeader(http.StatusAccepted) } - -// annotatedTag will annotate OCI tags by prepending a string, and leave docker -// tags unmodified. -func (imh *manifestHandler) annotatedTag(oci bool) string { - if oci { - return "oci." + imh.Tag - } - return imh.Tag -} From 9e3f78b8c84b21cb1f5774db6a34924209654f29 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Wed, 19 Jul 2017 13:29:10 -0500 Subject: [PATCH 07/20] addresses minor debug comments Signed-off-by: Mike Brown --- registry/handlers/manifests.go | 31 +++++++++----------------- registry/storage/ocimanifesthandler.go | 2 +- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index ae52bff9..2c6c7856 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -152,22 +152,22 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) isAnOCIImageIndex := isManifestList && (manifestList.MediaType == v1.MediaTypeImageIndex) if (isSchema2 && !isAnOCIManifest) && (supportsOCISchema && !supportsSchema2) { - fmt.Printf("\n\nmanifest is schema2, but accept header only supports OCISchema \n\n") + ctxu.GetLogger(imh).Debug("manifest is schema2, but accept header only supports OCISchema") w.WriteHeader(http.StatusNotFound) return } if (isManifestList && !isAnOCIImageIndex) && (supportsOCIImageIndex && !supportsManifestList) { - fmt.Printf("\n\nmanifestlist is not OCI, but accept header only supports an OCI manifestlist\n\n") + ctxu.GetLogger(imh).Debug("manifestlist is not OCI, but accept header only supports an OCI manifestlist") w.WriteHeader(http.StatusNotFound) return } if isAnOCIManifest && (!supportsOCISchema && supportsSchema2) { - fmt.Printf("\n\nmanifest is OCI, but accept header only supports schema2\n\n") + ctxu.GetLogger(imh).Debug("manifest is OCI, but accept header only supports schema2") w.WriteHeader(http.StatusNotFound) return } if isAnOCIImageIndex && (!supportsOCIImageIndex && supportsManifestList) { - fmt.Printf("\n\nmanifestlist is OCI, but accept header only supports non-OCI manifestlists\n\n") + ctxu.GetLogger(imh).Debug("manifestlist is OCI, but accept header only supports non-OCI manifestlists") w.WriteHeader(http.StatusNotFound) return } @@ -322,9 +322,9 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) isAnOCIManifest := mediaType == v1.MediaTypeImageManifest || mediaType == v1.MediaTypeImageIndex if isAnOCIManifest { - fmt.Printf("\n\nPutting an OCI Manifest!\n\n\n") + ctxu.GetLogger(imh).Debug("Putting an OCI Manifest!") } else { - fmt.Printf("\n\nPutting a Docker Manifest!\n\n\n") + ctxu.GetLogger(imh).Debug("Putting a Docker Manifest!") } var options []distribution.ManifestServiceOption @@ -342,12 +342,10 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) // TODO(stevvooe): These error handling switches really need to be // handled by an app global mapper. if err == distribution.ErrUnsupported { - fmt.Printf("\n\nXXX 1\n\n\n") imh.Errors = append(imh.Errors, errcode.ErrorCodeUnsupported) return } if err == distribution.ErrAccessDenied { - fmt.Printf("\n\nXXX 2\n\n\n") imh.Errors = append(imh.Errors, errcode.ErrorCodeDenied) return } @@ -374,7 +372,6 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) default: imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) } - fmt.Printf("\n\nXXX 3\n\n\n") return } @@ -383,7 +380,6 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) tags := imh.Repository.Tags(imh) err = tags.Tag(imh, imh.Tag, desc) if err != nil { - fmt.Printf("\n\nXXX 4: %T: %v\n\n\n", err, err) imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return } @@ -393,7 +389,6 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) // Construct a canonical url for the uploaded manifest. ref, err := reference.WithDigest(imh.Repository.Named(), imh.Digest) if err != nil { - fmt.Printf("\n\nXXX 5\n\n\n") imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err)) return } @@ -410,7 +405,7 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) w.Header().Set("Docker-Content-Digest", imh.Digest.String()) w.WriteHeader(http.StatusCreated) - fmt.Printf("\n\nSucceeded in putting manifest!\n\n\n") + ctxu.GetLogger(imh).Debug("Succeeded in putting manifest!") } // applyResourcePolicy checks whether the resource class matches what has @@ -432,16 +427,14 @@ func (imh *manifestHandler) applyResourcePolicy(manifest distribution.Manifest) case schema2.MediaTypePluginConfig: class = "plugin" default: - message := fmt.Sprintf("unknown manifest class for %s", m.Config.MediaType) - return errcode.ErrorCodeDenied.WithMessage(message) + return errcode.ErrorCodeDenied.WithMessage("unknown manifest class for " + m.Config.MediaType) } case *ocischema.DeserializedManifest: switch m.Config.MediaType { case v1.MediaTypeImageConfig: class = imageClass default: - message := fmt.Sprintf("unknown manifest class for %s", m.Config.MediaType) - return errcode.ErrorCodeDenied.WithMessage(message) + return errcode.ErrorCodeDenied.WithMessage("unknown manifest class for " + m.Config.MediaType) } } @@ -458,8 +451,7 @@ func (imh *manifestHandler) applyResourcePolicy(manifest distribution.Manifest) } } if !allowedClass { - message := fmt.Sprintf("registry does not allow %s manifest", class) - return errcode.ErrorCodeDenied.WithMessage(message) + return errcode.ErrorCodeDenied.WithMessage(fmt.Sprintf("registry does not allow %s manifest", class)) } resources := auth.AuthorizedResources(imh) @@ -480,8 +472,7 @@ func (imh *manifestHandler) applyResourcePolicy(manifest distribution.Manifest) // resource was found but no matching class was found if foundResource { - message := fmt.Sprintf("repository not authorized for %s manifest", class) - return errcode.ErrorCodeDenied.WithMessage(message) + return errcode.ErrorCodeDenied.WithMessage(fmt.Sprintf("repository not authorized for %s manifest", class)) } return nil diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index 148b7a2e..0efef56b 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -83,7 +83,7 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc // TODO: mikebrow/steveoe verify we should treat oci nondistributable like foreign layers? case v1.MediaTypeImageLayerNonDistributable, v1.MediaTypeImageLayerNonDistributableGzip: // Clients download this layer from an external URL, so do not check for - // its presense. + // its presence. if len(descriptor.URLs) == 0 { err = errMissingURL } From 6bae7ca5976ac55d8890a12c19e286ddc58a7719 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Thu, 20 Jul 2017 12:48:46 -0500 Subject: [PATCH 08/20] refactor adding enum for storage types Signed-off-by: Mike Brown --- manifest/manifestlist/manifestlist_test.go | 1 + manifest/ocischema/builder_test.go | 1 + manifest/ocischema/manifest.go | 1 - manifest/ocischema/manifest_test.go | 1 + registry/handlers/manifests.go | 67 ++++++++++++---------- registry/storage/driver/storagedriver.go | 2 +- 6 files changed, 41 insertions(+), 32 deletions(-) diff --git a/manifest/manifestlist/manifestlist_test.go b/manifest/manifestlist/manifestlist_test.go index 5ab328dd..6909895b 100644 --- a/manifest/manifestlist/manifestlist_test.go +++ b/manifest/manifestlist/manifestlist_test.go @@ -111,6 +111,7 @@ func TestManifestList(t *testing.T) { } } +// TODO (mikebrow): add annotations on the index and individual manifests var expectedOCIImageIndexSerialization = []byte(`{ "schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", diff --git a/manifest/ocischema/builder_test.go b/manifest/ocischema/builder_test.go index 33e9164f..ca785495 100644 --- a/manifest/ocischema/builder_test.go +++ b/manifest/ocischema/builder_test.go @@ -10,6 +10,7 @@ import ( "github.com/opencontainers/image-spec/specs-go/v1" ) +// TODO (not assigned): consider making mockBlobService common for ocischema, schema2, and schema1 type mockBlobService struct { descriptors map[digest.Digest]distribution.Descriptor } diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index a5b199d4..c3d9046a 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -57,7 +57,6 @@ func (m Manifest) References() []distribution.Descriptor { references := make([]distribution.Descriptor, 0, 1+len(m.Layers)) references = append(references, m.Config) references = append(references, m.Layers...) - // TODO: (mikebrow/stevvooe) should we return annotations as references return references } diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go index 55dc0306..714c50a5 100644 --- a/manifest/ocischema/manifest_test.go +++ b/manifest/ocischema/manifest_test.go @@ -10,6 +10,7 @@ import ( "github.com/opencontainers/image-spec/specs-go/v1" ) +// TODO (mikebrow): add annotations to the test var expectedManifestSerialization = []byte(`{ "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index 2c6c7856..ad39ccb5 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -30,6 +30,17 @@ const ( imageClass = "image" ) +type storageType int + +const ( + manifestSchema1 storageType = iota // 0 + manifestSchema2 // 1 + manifestlistSchema // 2 + ociSchema // 3 + ociImageIndexSchema // 4 + numStorageTypes // 5 +) + // manifestDispatcher takes the request context and builds the // appropriate handler for handling manifest requests. func manifestDispatcher(ctx *Context, r *http.Request) http.Handler { @@ -75,10 +86,8 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) imh.Errors = append(imh.Errors, err) return } - supportsSchema2 := false - supportsManifestList := false - supportsOCISchema := false - supportsOCIImageIndex := false + var supports [numStorageTypes]bool + // this parsing of Accept headers is not quite as full-featured as godoc.org's parser, but we don't care about "q=" values // https://github.com/golang/gddo/blob/e91d4165076d7474d20abda83f92d15c7ebc3e81/httputil/header/header.go#L165-L202 for _, acceptHeader := range r.Header["Accept"] { @@ -97,16 +106,16 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) mediaType = strings.TrimSpace(mediaType) if mediaType == schema2.MediaTypeManifest { - supportsSchema2 = true + supports[manifestSchema2] = true } if mediaType == manifestlist.MediaTypeManifestList { - supportsManifestList = true + supports[manifestlistSchema] = true } if mediaType == v1.MediaTypeImageManifest { - supportsOCISchema = true + supports[ociSchema] = true } if mediaType == v1.MediaTypeImageIndex { - supportsOCIImageIndex = true + supports[ociImageIndexSchema] = true } } } @@ -145,36 +154,34 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) } return } - + // determine the type of the returned manifest + manifestType := manifestSchema1 schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest) manifestList, isManifestList := manifest.(*manifestlist.DeserializedManifestList) - isAnOCIManifest := isSchema2 && (schema2Manifest.MediaType == v1.MediaTypeImageManifest) - isAnOCIImageIndex := isManifestList && (manifestList.MediaType == v1.MediaTypeImageIndex) + if isSchema2 { + manifestType = manifestSchema2 + } else if _, isOCImanifest := manifest.(*ocischema.DeserializedManifest); isOCImanifest { + manifestType = ociSchema + } else if isManifestList { + if manifestList.MediaType == manifestlist.MediaTypeManifestList { + manifestType = manifestlistSchema + } else if manifestList.MediaType == v1.MediaTypeImageIndex { + manifestType = ociImageIndexSchema + } + } - if (isSchema2 && !isAnOCIManifest) && (supportsOCISchema && !supportsSchema2) { - ctxu.GetLogger(imh).Debug("manifest is schema2, but accept header only supports OCISchema") - w.WriteHeader(http.StatusNotFound) + if manifestType == ociSchema && !supports[ociSchema] { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithMessage("OCI manifest found, but accept header does not support OCI manifests")) return } - if (isManifestList && !isAnOCIImageIndex) && (supportsOCIImageIndex && !supportsManifestList) { - ctxu.GetLogger(imh).Debug("manifestlist is not OCI, but accept header only supports an OCI manifestlist") - w.WriteHeader(http.StatusNotFound) - return - } - if isAnOCIManifest && (!supportsOCISchema && supportsSchema2) { - ctxu.GetLogger(imh).Debug("manifest is OCI, but accept header only supports schema2") - w.WriteHeader(http.StatusNotFound) - return - } - if isAnOCIImageIndex && (!supportsOCIImageIndex && supportsManifestList) { - ctxu.GetLogger(imh).Debug("manifestlist is OCI, but accept header only supports non-OCI manifestlists") - w.WriteHeader(http.StatusNotFound) + if manifestType == ociImageIndexSchema && !supports[ociImageIndexSchema] { + imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithMessage("OCI index found, but accept header does not support OCI indexes")) return } // Only rewrite schema2 manifests when they are being fetched by tag. // If they are being fetched by digest, we can't return something not // matching the digest. - if imh.Tag != "" && isSchema2 && !(supportsSchema2 || supportsOCISchema) { + if imh.Tag != "" && manifestType == manifestSchema2 && !supports[manifestSchema2] { // Rewrite manifest in schema1 format ctxu.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String()) @@ -182,7 +189,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) if err != nil { return } - } else if imh.Tag != "" && isManifestList && !(supportsManifestList || supportsOCIImageIndex) { + } else if imh.Tag != "" && manifestType == manifestlistSchema && !supports[manifestlistSchema] { // Rewrite manifest in schema1 format ctxu.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) @@ -212,7 +219,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) } // If necessary, convert the image manifest - if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !(supportsSchema2 || supportsOCISchema) { + if schema2Manifest, isSchema2 := manifest.(*schema2.DeserializedManifest); isSchema2 && !supports[manifestSchema2] { manifest, err = imh.convertSchema2Manifest(schema2Manifest) if err != nil { return diff --git a/registry/storage/driver/storagedriver.go b/registry/storage/driver/storagedriver.go index 46c1b9e0..b220713f 100644 --- a/registry/storage/driver/storagedriver.go +++ b/registry/storage/driver/storagedriver.go @@ -116,7 +116,7 @@ type FileWriter interface { // number of path components separated by slashes, where each component is // restricted to alphanumeric characters or a period, underscore, or // hyphen. -var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._:-]+)+$`) +var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._-]+)+$`) // ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method. type ErrUnsupportedMethod struct { From c1532332ad7fa88b692af45658e003e71a5a075f Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Thu, 20 Jul 2017 15:01:25 -0500 Subject: [PATCH 09/20] update to image spec v1.0.0 Signed-off-by: Mike Brown --- vendor.conf | 2 +- vendor/github.com/opencontainers/image-spec/specs-go/version.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor.conf b/vendor.conf index 4f52b597..df06259e 100644 --- a/vendor.conf +++ b/vendor.conf @@ -49,4 +49,4 @@ gopkg.in/square/go-jose.v1 40d457b439244b546f023d056628e5184136899b gopkg.in/yaml.v2 bef53efd0c76e49e6de55ead051f886bea7e9420 rsc.io/letsencrypt e770c10b0f1a64775ae91d240407ce00d1a5bdeb https://github.com/dmcgowan/letsencrypt.git github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb -github.com/opencontainers/image-spec e0536ae0cb47b8fed6afdaa9ba2589f6bd915caa +github.com/opencontainers/image-spec ab7389ef9f50030c9b245bc16b981c7ddf192882 diff --git a/vendor/github.com/opencontainers/image-spec/specs-go/version.go b/vendor/github.com/opencontainers/image-spec/specs-go/version.go index f4cda6ed..e3eee29b 100644 --- a/vendor/github.com/opencontainers/image-spec/specs-go/version.go +++ b/vendor/github.com/opencontainers/image-spec/specs-go/version.go @@ -25,7 +25,7 @@ const ( VersionPatch = 0 // VersionDev indicates development branch. Releases will be empty string. - VersionDev = "-rc6-dev" + VersionDev = "" ) // Version is the specification version that the package types support. From ec2aa05cdfa5b2318554841818f7b18d5be4582a Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Thu, 20 Jul 2017 20:44:02 -0500 Subject: [PATCH 10/20] addressing comments from stevvooe Signed-off-by: Mike Brown --- manifest/ocischema/builder_test.go | 150 +++++++------------- manifest/ocischema/manifest.go | 2 +- manifest/ocischema/manifest_test.go | 30 ++-- registry/handlers/manifests.go | 10 +- registry/storage/ocimanifesthandler.go | 18 --- registry/storage/ocimanifesthandler_test.go | 20 +-- 6 files changed, 86 insertions(+), 144 deletions(-) diff --git a/manifest/ocischema/builder_test.go b/manifest/ocischema/builder_test.go index ca785495..27562cfd 100644 --- a/manifest/ocischema/builder_test.go +++ b/manifest/ocischema/builder_test.go @@ -10,7 +10,6 @@ import ( "github.com/opencontainers/image-spec/specs-go/v1" ) -// TODO (not assigned): consider making mockBlobService common for ocischema, schema2, and schema1 type mockBlobService struct { descriptors map[digest.Digest]distribution.Descriptor } @@ -50,110 +49,65 @@ func (bs *mockBlobService) Resume(ctx context.Context, id string) (distribution. func TestBuilder(t *testing.T) { imgJSON := []byte(`{ + "created": "2015-10-31T22:22:56.015925234Z", + "author": "Alyssa P. Hacker ", "architecture": "amd64", - "config": { - "AttachStderr": false, - "AttachStdin": false, - "AttachStdout": false, - "Cmd": [ - "/bin/sh", - "-c", - "echo hi" - ], - "Domainname": "", - "Entrypoint": null, - "Env": [ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "derived=true", - "asdf=true" - ], - "Hostname": "23304fc829f9", - "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246", - "Labels": {}, - "OnBuild": [], - "OpenStdin": false, - "StdinOnce": false, - "Tty": false, - "User": "", - "Volumes": null, - "WorkingDir": "" - }, - "container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001", - "container_config": { - "AttachStderr": false, - "AttachStdin": false, - "AttachStdout": false, - "Cmd": [ - "/bin/sh", - "-c", - "#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]" - ], - "Domainname": "", - "Entrypoint": null, - "Env": [ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "derived=true", - "asdf=true" - ], - "Hostname": "23304fc829f9", - "Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246", - "Labels": {}, - "OnBuild": [], - "OpenStdin": false, - "StdinOnce": false, - "Tty": false, - "User": "", - "Volumes": null, - "WorkingDir": "" - }, - "created": "2015-11-04T23:06:32.365666163Z", - "docker_version": "1.9.0-dev", - "history": [ - { - "created": "2015-10-31T22:22:54.690851953Z", - "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /" - }, - { - "created": "2015-10-31T22:22:55.613815829Z", - "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]" - }, - { - "created": "2015-11-04T23:06:30.934316144Z", - "created_by": "/bin/sh -c #(nop) ENV derived=true", - "empty_layer": true - }, - { - "created": "2015-11-04T23:06:31.192097572Z", - "created_by": "/bin/sh -c #(nop) ENV asdf=true", - "empty_layer": true - }, - { - "created": "2015-11-04T23:06:32.083868454Z", - "created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024" - }, - { - "created": "2015-11-04T23:06:32.365666163Z", - "created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]", - "empty_layer": true - } - ], "os": "linux", - "rootfs": { - "diff_ids": [ - "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1", - "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef", - "sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49" + "config": { + "User": "alice", + "ExposedPorts": { + "8080/tcp": {} + }, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "FOO=oci_is_a", + "BAR=well_written_spec" ], - "type": "layers" - } + "Entrypoint": [ + "/bin/my-app-binary" + ], + "Cmd": [ + "--foreground", + "--config", + "/etc/my-app.d/default.cfg" + ], + "Volumes": { + "/var/job-result-data": {}, + "/var/log/my-app-logs": {} + }, + "WorkingDir": "/home/alice", + "Labels": { + "com.example.project.git.url": "https://example.com/project.git", + "com.example.project.git.commit": "45a939b2999782a3f005621a8d0f29aa387e1d6b" + } + }, + "rootfs": { + "diff_ids": [ + "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1", + "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" + ], + "type": "layers" + }, + "history": [ + { + "created": "2015-10-31T22:22:54.690851953Z", + "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /" + }, + { + "created": "2015-10-31T22:22:55.613815829Z", + "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]", + "empty_layer": true + } + ] }`) configDigest := digest.FromBytes(imgJSON) descriptors := []distribution.Descriptor{ { - Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"), - Size: 5312, - MediaType: v1.MediaTypeImageLayerGzip, + Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"), + Size: 5312, + MediaType: v1.MediaTypeImageLayerGzip, + Annotations: map[string]string{"apple": "orange", "lettuce": "wrap"}, }, { Digest: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"), @@ -200,7 +154,7 @@ func TestBuilder(t *testing.T) { if target.MediaType != v1.MediaTypeImageConfig { t.Fatalf("unexpected media type in target: %s", target.MediaType) } - if target.Size != 3153 { + if target.Size != 1582 { t.Fatalf("unexpected size in target: %d", target.Size) } diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index c3d9046a..afab12bd 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -15,7 +15,7 @@ var ( // SchemaVersion provides a pre-initialized version structure for this // packages version of the manifest. SchemaVersion = manifest.Versioned{ - SchemaVersion: 2, // TODO: (mikebrow/stevvooe) this could be confusing cause oci version 1 is closer to docker 2 than 1 + SchemaVersion: 2, // historical value here.. does not pertain to OCI or docker version MediaType: v1.MediaTypeImageManifest, } ) diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go index 714c50a5..c37fe8fb 100644 --- a/manifest/ocischema/manifest_test.go +++ b/manifest/ocischema/manifest_test.go @@ -17,13 +17,19 @@ var expectedManifestSerialization = []byte(`{ "config": { "mediaType": "application/vnd.oci.image.config.v1+json", "size": 985, - "digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" + "digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", + "annotations": { + "apple": "orange" + } }, "layers": [ { "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "size": 153263, - "digest": "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b" + "digest": "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b", + "annotations": { + "lettuce": "wrap" + } } ] }`) @@ -32,15 +38,17 @@ func TestManifest(t *testing.T) { manifest := Manifest{ Versioned: SchemaVersion, Config: distribution.Descriptor{ - Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", - Size: 985, - MediaType: v1.MediaTypeImageConfig, + Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", + Size: 985, + MediaType: v1.MediaTypeImageConfig, + Annotations: map[string]string{"apple": "orange"}, }, Layers: []distribution.Descriptor{ { - Digest: "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b", - Size: 153263, - MediaType: v1.MediaTypeImageLayerGzip, + Digest: "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b", + Size: 153263, + MediaType: v1.MediaTypeImageLayerGzip, + Annotations: map[string]string{"lettuce": "wrap"}, }, }, } @@ -90,6 +98,9 @@ func TestManifest(t *testing.T) { if target.Size != 985 { t.Fatalf("unexpected size in target: %d", target.Size) } + if target.Annotations["apple"] != "orange" { + t.Fatalf("unexpected annotation in target: %s", target.Annotations["apple"]) + } references := deserialized.References() if len(references) != 2 { @@ -110,4 +121,7 @@ func TestManifest(t *testing.T) { if references[1].Size != 153263 { t.Fatalf("unexpected size in reference: %d", references[0].Size) } + if references[1].Annotations["lettuce"] != "wrap" { + t.Fatalf("unexpected annotation in reference: %s", references[1].Annotations["lettuce"]) + } } diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index ad39ccb5..53c7c807 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -122,8 +122,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) if imh.Tag != "" { tags := imh.Repository.Tags(imh) - var desc distribution.Descriptor - desc, err = tags.Get(imh, imh.Tag) + desc, err := tags.Get(imh, imh.Tag) if err != nil { if _, ok := err.(distribution.ErrTagUnknown); ok { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) @@ -144,8 +143,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) if imh.Tag != "" { options = append(options, distribution.WithTag(imh.Tag)) } - var manifest distribution.Manifest - manifest, err = manifests.Get(imh, imh.Digest, options...) + manifest, err := manifests.Get(imh, imh.Digest, options...) if err != nil { if _, ok := err.(distribution.ErrManifestUnknownRevision); ok { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err)) @@ -266,7 +264,7 @@ func (imh *manifestHandler) convertSchema2Manifest(schema2Manifest *schema2.Dese builder := schema1.NewConfigManifestBuilder(imh.Repository.Blobs(imh), imh.Context.App.trustKey, ref, configJSON) for _, d := range schema2Manifest.Layers { - if err = builder.AppendReference(d); err != nil { + if err := builder.AppendReference(d); err != nil { imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err)) return nil, err } @@ -339,7 +337,7 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) options = append(options, distribution.WithTag(imh.Tag)) } - if err = imh.applyResourcePolicy(manifest); err != nil { + if err := imh.applyResourcePolicy(manifest); err != nil { imh.Errors = append(imh.Errors, err) return } diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index 0efef56b..c61beb4b 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -3,7 +3,6 @@ package storage import ( "encoding/json" "fmt" - "net/url" "github.com/docker/distribution" "github.com/docker/distribution/context" @@ -80,23 +79,6 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc var err error switch descriptor.MediaType { - // TODO: mikebrow/steveoe verify we should treat oci nondistributable like foreign layers? - case v1.MediaTypeImageLayerNonDistributable, v1.MediaTypeImageLayerNonDistributableGzip: - // Clients download this layer from an external URL, so do not check for - // its presence. - if len(descriptor.URLs) == 0 { - err = errMissingURL - } - allow := ms.manifestURLs.allow - deny := ms.manifestURLs.deny - for _, u := range descriptor.URLs { - var pu *url.URL - pu, err = url.Parse(u) - if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" || (allow != nil && !allow.MatchString(u)) || (deny != nil && deny.MatchString(u)) { - err = errInvalidURL - break - } - } case v1.MediaTypeImageManifest: var exists bool exists, err = manifestService.Exists(ctx, descriptor.Digest) diff --git a/registry/storage/ocimanifesthandler_test.go b/registry/storage/ocimanifesthandler_test.go index d04ce531..047f1c3a 100644 --- a/registry/storage/ocimanifesthandler_test.go +++ b/registry/storage/ocimanifesthandler_test.go @@ -53,12 +53,6 @@ func TestVerifyOCIManifestNonDistributableLayer(t *testing.T) { cases := []testcase{ { - nonDistributableLayer, - nil, - errMissingURL, - }, - { - // regular layers may have foreign urls (non-Distributable Layers) layer, []string{"http://foo/bar"}, nil, @@ -66,37 +60,37 @@ func TestVerifyOCIManifestNonDistributableLayer(t *testing.T) { { nonDistributableLayer, []string{"file:///local/file"}, - errInvalidURL, + nil, }, { nonDistributableLayer, []string{"http://foo/bar#baz"}, - errInvalidURL, + nil, }, { nonDistributableLayer, []string{""}, - errInvalidURL, + nil, }, { nonDistributableLayer, []string{"https://foo/bar", ""}, - errInvalidURL, + nil, }, { nonDistributableLayer, []string{"", "https://foo/bar"}, - errInvalidURL, + nil, }, { nonDistributableLayer, []string{"http://nope/bar"}, - errInvalidURL, + nil, }, { nonDistributableLayer, []string{"http://foo/nope"}, - errInvalidURL, + nil, }, { nonDistributableLayer, From f186e1da1cefaf5229f2d60d50b536078012e993 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Mon, 31 Jul 2017 18:34:11 -0500 Subject: [PATCH 11/20] adding some support for annotations at the manifest level Signed-off-by: Mike Brown --- manifest/ocischema/builder.go | 17 +++++++++++------ manifest/ocischema/builder_test.go | 11 +++++++++-- manifest/ocischema/manifest_test.go | 10 ++++++++-- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go index a6d6633a..c10376f8 100644 --- a/manifest/ocischema/builder.go +++ b/manifest/ocischema/builder.go @@ -18,15 +18,19 @@ type builder struct { // layers is a list of layer descriptors that gets built by successive // calls to AppendReference. layers []distribution.Descriptor + + // Annotations contains arbitrary metadata relating to the targeted content. + annotations map[string]string } // NewManifestBuilder is used to build new manifests for the current schema // version. It takes a BlobService so it can publish the configuration blob -// as part of the Build process. -func NewManifestBuilder(bs distribution.BlobService, configJSON []byte) distribution.ManifestBuilder { +// as part of the Build process, and annotations. +func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder { mb := &builder{ - bs: bs, - configJSON: make([]byte, len(configJSON)), + bs: bs, + configJSON: make([]byte, len(configJSON)), + annotations: annotations, } copy(mb.configJSON, configJSON) @@ -36,8 +40,9 @@ func NewManifestBuilder(bs distribution.BlobService, configJSON []byte) distribu // Build produces a final manifest from the given references. func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { m := Manifest{ - Versioned: SchemaVersion, - Layers: make([]distribution.Descriptor, len(mb.layers)), + Versioned: SchemaVersion, + Layers: make([]distribution.Descriptor, len(mb.layers)), + Annotations: mb.annotations, } copy(m.Layers, mb.layers) diff --git a/manifest/ocischema/builder_test.go b/manifest/ocischema/builder_test.go index 27562cfd..557681ba 100644 --- a/manifest/ocischema/builder_test.go +++ b/manifest/ocischema/builder_test.go @@ -88,6 +88,9 @@ func TestBuilder(t *testing.T) { ], "type": "layers" }, + "annotations": { + "hot": "potato" + } "history": [ { "created": "2015-10-31T22:22:54.690851953Z", @@ -120,9 +123,10 @@ func TestBuilder(t *testing.T) { MediaType: v1.MediaTypeImageLayerGzip, }, } + annotations := map[string]string{"hot": "potato"} bs := &mockBlobService{descriptors: make(map[digest.Digest]distribution.Descriptor)} - builder := NewManifestBuilder(bs, imgJSON) + builder := NewManifestBuilder(bs, imgJSON, annotations) for _, d := range descriptors { if err := builder.AppendReference(d); err != nil { @@ -142,6 +146,9 @@ func TestBuilder(t *testing.T) { } manifest := built.(*DeserializedManifest).Manifest + if manifest.Annotations["hot"] != "potato" { + t.Fatalf("unexpected annotation in manifest: %s", manifest.Annotations["hot"]) + } if manifest.Versioned.SchemaVersion != 2 { t.Fatal("SchemaVersion != 2") @@ -154,7 +161,7 @@ func TestBuilder(t *testing.T) { if target.MediaType != v1.MediaTypeImageConfig { t.Fatalf("unexpected media type in target: %s", target.MediaType) } - if target.Size != 1582 { + if target.Size != 1632 { t.Fatalf("unexpected size in target: %d", target.Size) } diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go index c37fe8fb..749cb859 100644 --- a/manifest/ocischema/manifest_test.go +++ b/manifest/ocischema/manifest_test.go @@ -10,7 +10,6 @@ import ( "github.com/opencontainers/image-spec/specs-go/v1" ) -// TODO (mikebrow): add annotations to the test var expectedManifestSerialization = []byte(`{ "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", @@ -31,7 +30,10 @@ var expectedManifestSerialization = []byte(`{ "lettuce": "wrap" } } - ] + ], + "annotations": { + "hot": "potato" + } }`) func TestManifest(t *testing.T) { @@ -51,6 +53,7 @@ func TestManifest(t *testing.T) { Annotations: map[string]string{"lettuce": "wrap"}, }, }, + Annotations: map[string]string{"hot": "potato"}, } deserialized, err := FromStruct(manifest) @@ -87,6 +90,9 @@ func TestManifest(t *testing.T) { if !reflect.DeepEqual(&unmarshalled, deserialized) { t.Fatalf("manifests are different after unmarshaling: %v != %v", unmarshalled, *deserialized) } + if deserialized.Annotations["hot"] != "potato" { + t.Fatalf("unexpected annotation in manifest: %s", deserialized.Annotations["hot"]) + } target := deserialized.Target() if target.Digest != "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" { From 7b47fb13cfdf637218cb642121539f4e17c5afd7 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Tue, 1 Aug 2017 17:27:52 -0500 Subject: [PATCH 12/20] update url policy support; testing for annoations in index Signed-off-by: Mike Brown --- manifest/manifestlist/manifestlist.go | 3 -- manifest/manifestlist/manifestlist_test.go | 41 ++++++++++++++++++--- registry/storage/ocimanifesthandler.go | 17 +++++++++ registry/storage/ocimanifesthandler_test.go | 21 +++++++---- 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/manifest/manifestlist/manifestlist.go b/manifest/manifestlist/manifestlist.go index 3303c6be..0d5c9af1 100644 --- a/manifest/manifestlist/manifestlist.go +++ b/manifest/manifestlist/manifestlist.go @@ -89,9 +89,6 @@ type ManifestList struct { // Config references the image configuration as a blob. Manifests []ManifestDescriptor `json:"manifests"` - - // Annotations contains arbitrary metadata for the image index. - Annotations map[string]string `json:"annotations,omitempty"` } // References returns the distribution descriptors for the referenced image diff --git a/manifest/manifestlist/manifestlist_test.go b/manifest/manifestlist/manifestlist_test.go index 6909895b..720b911b 100644 --- a/manifest/manifestlist/manifestlist_test.go +++ b/manifest/manifestlist/manifestlist_test.go @@ -111,7 +111,12 @@ func TestManifestList(t *testing.T) { } } -// TODO (mikebrow): add annotations on the index and individual manifests +// TODO (mikebrow): add annotations on the manifest list (index) and support for +// empty platform structs (move to Platform *Platform `json:"platform,omitempty"` +// from current Platform PlatformSpec `json:"platform"`) in the manifest descriptor. +// Requires changes to docker/distribution/manifest/manifestlist.ManifestList and .ManifestDescriptor +// and associated serialization APIs in manifestlist.go. Or split the OCI index and +// docker manifest list implementations, which would require a lot of refactoring. var expectedOCIImageIndexSerialization = []byte(`{ "schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", @@ -128,10 +133,25 @@ var expectedOCIImageIndexSerialization = []byte(`{ ] } }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 985, + "digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", + "annotations": { + "platform": "none" + }, + "platform": { + "architecture": "", + "os": "" + } + }, { "mediaType": "application/vnd.oci.image.manifest.v1+json", "size": 2392, "digest": "sha256:6346340964309634683409684360934680934608934608934608934068934608", + "annotations": { + "what": "for" + }, "platform": { "architecture": "sun4m", "os": "sunos" @@ -156,9 +176,18 @@ func TestOCIImageIndex(t *testing.T) { }, { Descriptor: distribution.Descriptor{ - Digest: "sha256:6346340964309634683409684360934680934608934608934608934068934608", - Size: 2392, - MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", + Size: 985, + MediaType: "application/vnd.oci.image.manifest.v1+json", + Annotations: map[string]string{"platform": "none"}, + }, + }, + { + Descriptor: distribution.Descriptor{ + Digest: "sha256:6346340964309634683409684360934680934608934608934608934068934608", + Size: 2392, + MediaType: "application/vnd.oci.image.manifest.v1+json", + Annotations: map[string]string{"what": "for"}, }, Platform: PlatformSpec{ Architecture: "sun4m", @@ -190,7 +219,7 @@ func TestOCIImageIndex(t *testing.T) { // Check that the canonical field has the expected value. if !bytes.Equal(expectedOCIImageIndexSerialization, canonical) { - t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedOCIImageIndexSerialization)) + t.Fatalf("manifest bytes not equal to expected: %q != %q", string(canonical), string(expectedOCIImageIndexSerialization)) } var unmarshalled DeserializedManifestList @@ -203,7 +232,7 @@ func TestOCIImageIndex(t *testing.T) { } references := deserialized.References() - if len(references) != 2 { + if len(references) != 3 { t.Fatalf("unexpected number of references: %d", len(references)) } for i := range references { diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index c61beb4b..f1fbcead 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -3,6 +3,7 @@ package storage import ( "encoding/json" "fmt" + "net/url" "github.com/docker/distribution" "github.com/docker/distribution/context" @@ -79,6 +80,22 @@ func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst oc var err error switch descriptor.MediaType { + case v1.MediaTypeImageLayer, v1.MediaTypeImageLayerGzip, v1.MediaTypeImageLayerNonDistributable, v1.MediaTypeImageLayerNonDistributableGzip: + allow := ms.manifestURLs.allow + deny := ms.manifestURLs.deny + for _, u := range descriptor.URLs { + var pu *url.URL + pu, err = url.Parse(u) + if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" || (allow != nil && !allow.MatchString(u)) || (deny != nil && deny.MatchString(u)) { + err = errInvalidURL + break + } + } + if err == nil && len(descriptor.URLs) == 0 { + // If no URLs, require that the blob exists + _, err = blobsService.Stat(ctx, descriptor.Digest) + } + case v1.MediaTypeImageManifest: var exists bool exists, err = manifestService.Exists(ctx, descriptor.Digest) diff --git a/registry/storage/ocimanifesthandler_test.go b/registry/storage/ocimanifesthandler_test.go index 047f1c3a..302294af 100644 --- a/registry/storage/ocimanifesthandler_test.go +++ b/registry/storage/ocimanifesthandler_test.go @@ -52,6 +52,11 @@ func TestVerifyOCIManifestNonDistributableLayer(t *testing.T) { } cases := []testcase{ + { + nonDistributableLayer, + nil, + distribution.ErrManifestBlobUnknown{Digest: nonDistributableLayer.Digest}, + }, { layer, []string{"http://foo/bar"}, @@ -60,37 +65,37 @@ func TestVerifyOCIManifestNonDistributableLayer(t *testing.T) { { nonDistributableLayer, []string{"file:///local/file"}, - nil, + errInvalidURL, }, { nonDistributableLayer, []string{"http://foo/bar#baz"}, - nil, + errInvalidURL, }, { nonDistributableLayer, []string{""}, - nil, + errInvalidURL, }, { nonDistributableLayer, []string{"https://foo/bar", ""}, - nil, + errInvalidURL, }, { nonDistributableLayer, []string{"", "https://foo/bar"}, - nil, + errInvalidURL, }, { nonDistributableLayer, []string{"http://nope/bar"}, - nil, + errInvalidURL, }, { nonDistributableLayer, []string{"http://foo/nope"}, - nil, + errInvalidURL, }, { nonDistributableLayer, @@ -122,6 +127,8 @@ func TestVerifyOCIManifestNonDistributableLayer(t *testing.T) { if _, ok = verr[1].(distribution.ErrManifestBlobUnknown); ok { err = verr[0] } + } else if len(verr) == 1 { + err = verr[0] } } if err != c.Err { From ad7ab0853cca5599ddf7f06bbdd76b823d23cf63 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Thu, 31 Aug 2017 21:14:40 -0500 Subject: [PATCH 13/20] folow commit 9c88801a12a97de49abf26e9604975eececa654c Signed-off-by: Mike Brown --- manifest/ocischema/builder.go | 3 ++- manifest/ocischema/builder_test.go | 2 +- registry/handlers/manifests.go | 10 +++++----- registry/storage/ocimanifesthandler.go | 9 +++++---- registry/storage/ocimanifesthandler_test.go | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go index c10376f8..9bfef614 100644 --- a/manifest/ocischema/builder.go +++ b/manifest/ocischema/builder.go @@ -1,8 +1,9 @@ package ocischema import ( + "context" + "github.com/docker/distribution" - "github.com/docker/distribution/context" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go/v1" ) diff --git a/manifest/ocischema/builder_test.go b/manifest/ocischema/builder_test.go index 557681ba..8e751279 100644 --- a/manifest/ocischema/builder_test.go +++ b/manifest/ocischema/builder_test.go @@ -1,11 +1,11 @@ package ocischema import ( + "context" "reflect" "testing" "github.com/docker/distribution" - "github.com/docker/distribution/context" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go/v1" ) diff --git a/registry/handlers/manifests.go b/registry/handlers/manifests.go index 53c7c807..8da3f7af 100644 --- a/registry/handlers/manifests.go +++ b/registry/handlers/manifests.go @@ -181,7 +181,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) // matching the digest. if imh.Tag != "" && manifestType == manifestSchema2 && !supports[manifestSchema2] { // Rewrite manifest in schema1 format - ctxu.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String()) + dcontext.GetLogger(imh).Infof("rewriting manifest %s in schema1 format to support old client", imh.Digest.String()) manifest, err = imh.convertSchema2Manifest(schema2Manifest) if err != nil { @@ -189,7 +189,7 @@ func (imh *manifestHandler) GetManifest(w http.ResponseWriter, r *http.Request) } } else if imh.Tag != "" && manifestType == manifestlistSchema && !supports[manifestlistSchema] { // Rewrite manifest in schema1 format - ctxu.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) + dcontext.GetLogger(imh).Infof("rewriting manifest list %s in schema1 format to support old client", imh.Digest.String()) // Find the image manifest corresponding to the default // platform @@ -327,9 +327,9 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) isAnOCIManifest := mediaType == v1.MediaTypeImageManifest || mediaType == v1.MediaTypeImageIndex if isAnOCIManifest { - ctxu.GetLogger(imh).Debug("Putting an OCI Manifest!") + dcontext.GetLogger(imh).Debug("Putting an OCI Manifest!") } else { - ctxu.GetLogger(imh).Debug("Putting a Docker Manifest!") + dcontext.GetLogger(imh).Debug("Putting a Docker Manifest!") } var options []distribution.ManifestServiceOption @@ -410,7 +410,7 @@ func (imh *manifestHandler) PutManifest(w http.ResponseWriter, r *http.Request) w.Header().Set("Docker-Content-Digest", imh.Digest.String()) w.WriteHeader(http.StatusCreated) - ctxu.GetLogger(imh).Debug("Succeeded in putting manifest!") + dcontext.GetLogger(imh).Debug("Succeeded in putting manifest!") } // applyResourcePolicy checks whether the resource class matches what has diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go index f1fbcead..57ff2829 100644 --- a/registry/storage/ocimanifesthandler.go +++ b/registry/storage/ocimanifesthandler.go @@ -1,12 +1,13 @@ package storage import ( + "context" "encoding/json" "fmt" "net/url" "github.com/docker/distribution" - "github.com/docker/distribution/context" + dcontext "github.com/docker/distribution/context" "github.com/docker/distribution/manifest/ocischema" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go/v1" @@ -23,7 +24,7 @@ type ocischemaManifestHandler struct { var _ ManifestHandler = &ocischemaManifestHandler{} func (ms *ocischemaManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { - context.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Unmarshal") + dcontext.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Unmarshal") var m ocischema.DeserializedManifest if err := json.Unmarshal(content, &m); err != nil { @@ -34,7 +35,7 @@ func (ms *ocischemaManifestHandler) Unmarshal(ctx context.Context, dgst digest.D } func (ms *ocischemaManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { - context.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Put") + dcontext.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Put") m, ok := manifest.(*ocischema.DeserializedManifest) if !ok { @@ -52,7 +53,7 @@ func (ms *ocischemaManifestHandler) Put(ctx context.Context, manifest distributi revision, err := ms.blobStore.Put(ctx, mt, payload) if err != nil { - context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + dcontext.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) return "", err } diff --git a/registry/storage/ocimanifesthandler_test.go b/registry/storage/ocimanifesthandler_test.go index 302294af..3983a9f7 100644 --- a/registry/storage/ocimanifesthandler_test.go +++ b/registry/storage/ocimanifesthandler_test.go @@ -1,11 +1,11 @@ package storage import ( + "context" "regexp" "testing" "github.com/docker/distribution" - "github.com/docker/distribution/context" "github.com/docker/distribution/manifest" "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/registry/storage/driver/inmemory" From f6224f78ba1c0b5e5a71678fd9d493465ddd5c9d Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Mon, 18 Jun 2018 15:18:59 -0500 Subject: [PATCH 14/20] register oci image index Signed-off-by: Mike Brown --- manifest/manifestlist/manifestlist.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/manifest/manifestlist/manifestlist.go b/manifest/manifestlist/manifestlist.go index 0d5c9af1..7be29714 100644 --- a/manifest/manifestlist/manifestlist.go +++ b/manifest/manifestlist/manifestlist.go @@ -45,6 +45,21 @@ func init() { if err != nil { panic(fmt.Sprintf("Unable to register manifest: %s", err)) } + + imageIndexFunc := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) { + m := new(DeserializedManifestList) + err := m.UnmarshalJSON(b) + if err != nil { + return nil, distribution.Descriptor{}, err + } + + dgst := digest.FromBytes(b) + return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageIndex}, err + } + err = distribution.RegisterManifestSchema(v1.MediaTypeImageIndex, imageIndexFunc) + if err != nil { + panic(fmt.Sprintf("Unable to register OCI Image Index: %s", err)) + } } // PlatformSpec specifies a platform where a particular image manifest is From 1d47ef7b801369dc2ca257732ea91d39089ecebc Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Wed, 14 Mar 2018 16:55:45 -0400 Subject: [PATCH 15/20] Check media types when unmarshalling manifests When unmarshalling manifests from JSON, check that the MediaType field corresponds to the type that we are unmarshalling as. This makes sure that when we retrieve a manifest from the manifest store, it will have the same type as it was handled as before storing it in the manifest store. Signed-off-by: Owen W. Taylor --- manifest/manifestlist/manifestlist.go | 36 ++++++++--- manifest/manifestlist/manifestlist_test.go | 70 ++++++++++++++++++++-- manifest/ocischema/manifest.go | 5 ++ manifest/ocischema/manifest_test.go | 57 +++++++++++++++++- manifest/schema2/manifest.go | 6 ++ manifest/schema2/manifest_test.go | 57 +++++++++++++++++- 6 files changed, 214 insertions(+), 17 deletions(-) diff --git a/manifest/manifestlist/manifestlist.go b/manifest/manifestlist/manifestlist.go index 7be29714..11df4c25 100644 --- a/manifest/manifestlist/manifestlist.go +++ b/manifest/manifestlist/manifestlist.go @@ -38,6 +38,13 @@ func init() { return nil, distribution.Descriptor{}, err } + if m.MediaType != MediaTypeManifestList { + err = fmt.Errorf("mediaType in manifest list should be '%s' not '%s'", + MediaTypeManifestList, m.MediaType) + + return nil, distribution.Descriptor{}, err + } + dgst := digest.FromBytes(b) return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifestList}, err } @@ -53,6 +60,13 @@ func init() { return nil, distribution.Descriptor{}, err } + if m.MediaType != v1.MediaTypeImageIndex { + err = fmt.Errorf("mediaType in image index should be '%s' not '%s'", + v1.MediaTypeImageIndex, m.MediaType) + + return nil, distribution.Descriptor{}, err + } + dgst := digest.FromBytes(b) return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: v1.MediaTypeImageIndex}, err } @@ -130,15 +144,23 @@ type DeserializedManifestList struct { // DeserializedManifestList which contains the resulting manifest list // and its JSON representation. func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestList, error) { - var m ManifestList + var mediaType string if len(descriptors) > 0 && descriptors[0].Descriptor.MediaType == v1.MediaTypeImageManifest { - m = ManifestList{ - Versioned: OCISchemaVersion, - } + mediaType = v1.MediaTypeImageIndex } else { - m = ManifestList{ - Versioned: SchemaVersion, - } + mediaType = MediaTypeManifestList + } + + return FromDescriptorsWithMediaType(descriptors, mediaType) +} + +// For testing purposes, it's useful to be able to specify the media type explicitly +func FromDescriptorsWithMediaType(descriptors []ManifestDescriptor, mediaType string) (*DeserializedManifestList, error) { + m := ManifestList{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mediaType, + }, } m.Manifests = make([]ManifestDescriptor, len(descriptors), len(descriptors)) diff --git a/manifest/manifestlist/manifestlist_test.go b/manifest/manifestlist/manifestlist_test.go index 720b911b..00b43900 100644 --- a/manifest/manifestlist/manifestlist_test.go +++ b/manifest/manifestlist/manifestlist_test.go @@ -38,7 +38,7 @@ var expectedManifestListSerialization = []byte(`{ ] }`) -func TestManifestList(t *testing.T) { +func makeTestManifestList(t *testing.T, mediaType string) ([]ManifestDescriptor, *DeserializedManifestList) { manifestDescriptors := []ManifestDescriptor{ { Descriptor: distribution.Descriptor{ @@ -65,11 +65,16 @@ func TestManifestList(t *testing.T) { }, } - deserialized, err := FromDescriptors(manifestDescriptors) + deserialized, err := FromDescriptorsWithMediaType(manifestDescriptors, mediaType) if err != nil { t.Fatalf("error creating DeserializedManifestList: %v", err) } + return manifestDescriptors, deserialized +} + +func TestManifestList(t *testing.T) { + manifestDescriptors, deserialized := makeTestManifestList(t, MediaTypeManifestList) mediaType, canonical, _ := deserialized.Payload() if mediaType != MediaTypeManifestList { @@ -160,7 +165,7 @@ var expectedOCIImageIndexSerialization = []byte(`{ ] }`) -func TestOCIImageIndex(t *testing.T) { +func makeTestOCIImageIndex(t *testing.T, mediaType string) ([]ManifestDescriptor, *DeserializedManifestList) { manifestDescriptors := []ManifestDescriptor{ { Descriptor: distribution.Descriptor{ @@ -196,11 +201,17 @@ func TestOCIImageIndex(t *testing.T) { }, } - deserialized, err := FromDescriptors(manifestDescriptors) + deserialized, err := FromDescriptorsWithMediaType(manifestDescriptors, mediaType) if err != nil { t.Fatalf("error creating DeserializedManifestList: %v", err) } + return manifestDescriptors, deserialized +} + +func TestOCIImageIndex(t *testing.T) { + manifestDescriptors, deserialized := makeTestOCIImageIndex(t, v1.MediaTypeImageIndex) + mediaType, canonical, _ := deserialized.Payload() if mediaType != v1.MediaTypeImageIndex { @@ -241,3 +252,54 @@ func TestOCIImageIndex(t *testing.T) { } } } + +func mediaTypeTest(t *testing.T, contentType string, mediaType string, shouldError bool) { + var m *DeserializedManifestList + if contentType == MediaTypeManifestList { + _, m = makeTestManifestList(t, mediaType) + } else { + _, m = makeTestOCIImageIndex(t, mediaType) + } + + _, canonical, err := m.Payload() + if err != nil { + t.Fatalf("error getting payload, %v", err) + } + + unmarshalled, descriptor, err := distribution.UnmarshalManifest( + contentType, + canonical) + + if shouldError { + if err == nil { + t.Fatalf("bad content type should have produced error") + } + } else { + if err != nil { + t.Fatalf("error unmarshaling manifest, %v", err) + } + + asManifest := unmarshalled.(*DeserializedManifestList) + if asManifest.MediaType != mediaType { + t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType) + } + + if descriptor.MediaType != contentType { + t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType) + } + + unmarshalledMediaType, _, _ := unmarshalled.Payload() + if unmarshalledMediaType != contentType { + t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType) + } + } +} + +func TestMediaTypes(t *testing.T) { + mediaTypeTest(t, MediaTypeManifestList, "", true) + mediaTypeTest(t, MediaTypeManifestList, MediaTypeManifestList, false) + mediaTypeTest(t, MediaTypeManifestList, MediaTypeManifestList+"XXX", true) + mediaTypeTest(t, v1.MediaTypeImageIndex, "", true) + mediaTypeTest(t, v1.MediaTypeImageIndex, v1.MediaTypeImageIndex, false) + mediaTypeTest(t, v1.MediaTypeImageIndex, v1.MediaTypeImageIndex+"XXX", true) +} diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index afab12bd..a6850fde 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -97,6 +97,11 @@ func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { return err } + if manifest.MediaType != v1.MediaTypeImageManifest { + return fmt.Errorf("mediaType in manifest should be '%s' not '%s'", + v1.MediaTypeImageManifest, manifest.MediaType) + } + m.Manifest = manifest return nil diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go index 749cb859..de53806d 100644 --- a/manifest/ocischema/manifest_test.go +++ b/manifest/ocischema/manifest_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/docker/distribution" + "github.com/docker/distribution/manifest" "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -36,9 +37,12 @@ var expectedManifestSerialization = []byte(`{ } }`) -func TestManifest(t *testing.T) { - manifest := Manifest{ - Versioned: SchemaVersion, +func makeTestManifest(mediaType string) Manifest { + return Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mediaType, + }, Config: distribution.Descriptor{ Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", Size: 985, @@ -55,6 +59,10 @@ func TestManifest(t *testing.T) { }, Annotations: map[string]string{"hot": "potato"}, } +} + +func TestManifest(t *testing.T) { + manifest := makeTestManifest(v1.MediaTypeImageManifest) deserialized, err := FromStruct(manifest) if err != nil { @@ -131,3 +139,46 @@ func TestManifest(t *testing.T) { t.Fatalf("unexpected annotation in reference: %s", references[1].Annotations["lettuce"]) } } + +func mediaTypeTest(t *testing.T, mediaType string, shouldError bool) { + manifest := makeTestManifest(mediaType) + + deserialized, err := FromStruct(manifest) + if err != nil { + t.Fatalf("error creating DeserializedManifest: %v", err) + } + + unmarshalled, descriptor, err := distribution.UnmarshalManifest( + v1.MediaTypeImageManifest, + deserialized.canonical) + + if shouldError { + if err == nil { + t.Fatalf("bad content type should have produced error") + } + } else { + if err != nil { + t.Fatalf("error unmarshaling manifest, %v", err) + } + + asManifest := unmarshalled.(*DeserializedManifest) + if asManifest.MediaType != mediaType { + t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType) + } + + if descriptor.MediaType != v1.MediaTypeImageManifest { + t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType) + } + + unmarshalledMediaType, _, _ := unmarshalled.Payload() + if unmarshalledMediaType != v1.MediaTypeImageManifest { + t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType) + } + } +} + +func TestMediaTypes(t *testing.T) { + mediaTypeTest(t, "", true) + mediaTypeTest(t, v1.MediaTypeImageManifest, false) + mediaTypeTest(t, v1.MediaTypeImageManifest+"XXX", true) +} diff --git a/manifest/schema2/manifest.go b/manifest/schema2/manifest.go index 26a6a334..8adbbd0f 100644 --- a/manifest/schema2/manifest.go +++ b/manifest/schema2/manifest.go @@ -116,6 +116,12 @@ func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { return err } + if manifest.MediaType != MediaTypeManifest { + return fmt.Errorf("mediaType in manifest should be '%s' not '%s'", + MediaTypeManifest, manifest.MediaType) + + } + m.Manifest = manifest return nil diff --git a/manifest/schema2/manifest_test.go b/manifest/schema2/manifest_test.go index 86226606..912dda9a 100644 --- a/manifest/schema2/manifest_test.go +++ b/manifest/schema2/manifest_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/docker/distribution" + "github.com/docker/distribution/manifest" ) var expectedManifestSerialization = []byte(`{ @@ -26,9 +27,12 @@ var expectedManifestSerialization = []byte(`{ ] }`) -func TestManifest(t *testing.T) { - manifest := Manifest{ - Versioned: SchemaVersion, +func makeTestManifest(mediaType string) Manifest { + return Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mediaType, + }, Config: distribution.Descriptor{ Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b", Size: 985, @@ -42,6 +46,10 @@ func TestManifest(t *testing.T) { }, }, } +} + +func TestManifest(t *testing.T) { + manifest := makeTestManifest(MediaTypeManifest) deserialized, err := FromStruct(manifest) if err != nil { @@ -109,3 +117,46 @@ func TestManifest(t *testing.T) { t.Fatalf("unexpected size in reference: %d", references[0].Size) } } + +func mediaTypeTest(t *testing.T, mediaType string, shouldError bool) { + manifest := makeTestManifest(mediaType) + + deserialized, err := FromStruct(manifest) + if err != nil { + t.Fatalf("error creating DeserializedManifest: %v", err) + } + + unmarshalled, descriptor, err := distribution.UnmarshalManifest( + MediaTypeManifest, + deserialized.canonical) + + if shouldError { + if err == nil { + t.Fatalf("bad content type should have produced error") + } + } else { + if err != nil { + t.Fatalf("error unmarshaling manifest, %v", err) + } + + asManifest := unmarshalled.(*DeserializedManifest) + if asManifest.MediaType != mediaType { + t.Fatalf("Bad media type '%v' as unmarshalled", asManifest.MediaType) + } + + if descriptor.MediaType != MediaTypeManifest { + t.Fatalf("Bad media type '%v' for descriptor", descriptor.MediaType) + } + + unmarshalledMediaType, _, _ := unmarshalled.Payload() + if unmarshalledMediaType != MediaTypeManifest { + t.Fatalf("Bad media type '%v' for payload", unmarshalledMediaType) + } + } +} + +func TestMediaTypes(t *testing.T) { + mediaTypeTest(t, "", true) + mediaTypeTest(t, MediaTypeManifest, false) + mediaTypeTest(t, MediaTypeManifest+"XXX", true) +} From 60d9c5dfad8c0d44fd5ee0482f6de7b09a15fb02 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Wed, 14 Mar 2018 17:22:36 -0400 Subject: [PATCH 16/20] Handle OCI manifests and image indexes without a media type In the OCI image specification, the MediaType field is reserved and otherwise undefined; assume that manifests without a media in storage are OCI images or image indexes, and determine which by looking at what fields are in the JSON. We do keep a check that when unmarshalling an OCI image or image index, if it has a MediaType field, it must match that media type of the upload. Signed-off-by: Owen W. Taylor --- manifest/manifestlist/manifestlist.go | 13 ++++++++++--- manifest/manifestlist/manifestlist_test.go | 2 +- manifest/ocischema/manifest.go | 6 +++--- manifest/ocischema/manifest_test.go | 2 +- registry/storage/manifeststore.go | 12 ++++++++++++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/manifest/manifestlist/manifestlist.go b/manifest/manifestlist/manifestlist.go index 11df4c25..a08f1e5a 100644 --- a/manifest/manifestlist/manifestlist.go +++ b/manifest/manifestlist/manifestlist.go @@ -60,8 +60,8 @@ func init() { return nil, distribution.Descriptor{}, err } - if m.MediaType != v1.MediaTypeImageIndex { - err = fmt.Errorf("mediaType in image index should be '%s' not '%s'", + if m.MediaType != "" && m.MediaType != v1.MediaTypeImageIndex { + err = fmt.Errorf("if present, mediaType in image index should be '%s' not '%s'", v1.MediaTypeImageIndex, m.MediaType) return nil, distribution.Descriptor{}, err @@ -205,5 +205,12 @@ func (m *DeserializedManifestList) MarshalJSON() ([]byte, error) { // Payload returns the raw content of the manifest list. The contents can be // used to calculate the content identifier. func (m DeserializedManifestList) Payload() (string, []byte, error) { - return m.MediaType, m.canonical, nil + var mediaType string + if m.MediaType == "" { + mediaType = v1.MediaTypeImageIndex + } else { + mediaType = m.MediaType + } + + return mediaType, m.canonical, nil } diff --git a/manifest/manifestlist/manifestlist_test.go b/manifest/manifestlist/manifestlist_test.go index 00b43900..e72292f0 100644 --- a/manifest/manifestlist/manifestlist_test.go +++ b/manifest/manifestlist/manifestlist_test.go @@ -299,7 +299,7 @@ func TestMediaTypes(t *testing.T) { mediaTypeTest(t, MediaTypeManifestList, "", true) mediaTypeTest(t, MediaTypeManifestList, MediaTypeManifestList, false) mediaTypeTest(t, MediaTypeManifestList, MediaTypeManifestList+"XXX", true) - mediaTypeTest(t, v1.MediaTypeImageIndex, "", true) + mediaTypeTest(t, v1.MediaTypeImageIndex, "", false) mediaTypeTest(t, v1.MediaTypeImageIndex, v1.MediaTypeImageIndex, false) mediaTypeTest(t, v1.MediaTypeImageIndex, v1.MediaTypeImageIndex+"XXX", true) } diff --git a/manifest/ocischema/manifest.go b/manifest/ocischema/manifest.go index a6850fde..6de17f9a 100644 --- a/manifest/ocischema/manifest.go +++ b/manifest/ocischema/manifest.go @@ -97,8 +97,8 @@ func (m *DeserializedManifest) UnmarshalJSON(b []byte) error { return err } - if manifest.MediaType != v1.MediaTypeImageManifest { - return fmt.Errorf("mediaType in manifest should be '%s' not '%s'", + if manifest.MediaType != "" && manifest.MediaType != v1.MediaTypeImageManifest { + return fmt.Errorf("if present, mediaType in manifest should be '%s' not '%s'", v1.MediaTypeImageManifest, manifest.MediaType) } @@ -120,5 +120,5 @@ func (m *DeserializedManifest) MarshalJSON() ([]byte, error) { // Payload returns the raw content of the manifest. The contents can be used to // calculate the content identifier. func (m DeserializedManifest) Payload() (string, []byte, error) { - return m.MediaType, m.canonical, nil + return v1.MediaTypeImageManifest, m.canonical, nil } diff --git a/manifest/ocischema/manifest_test.go b/manifest/ocischema/manifest_test.go index de53806d..c39d1c9f 100644 --- a/manifest/ocischema/manifest_test.go +++ b/manifest/ocischema/manifest_test.go @@ -178,7 +178,7 @@ func mediaTypeTest(t *testing.T, mediaType string, shouldError bool) { } func TestMediaTypes(t *testing.T) { - mediaTypeTest(t, "", true) + mediaTypeTest(t, "", false) mediaTypeTest(t, v1.MediaTypeImageManifest, false) mediaTypeTest(t, v1.MediaTypeImageManifest+"XXX", true) } diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 6543f6fd..73bff573 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -106,6 +106,18 @@ func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options .. return ms.ocischemaHandler.Unmarshal(ctx, dgst, content) case manifestlist.MediaTypeManifestList, v1.MediaTypeImageIndex: return ms.manifestListHandler.Unmarshal(ctx, dgst, content) + case "": + // OCI image or image index - no media type in the content + + // First see if it looks like an image index + res, err := ms.manifestListHandler.Unmarshal(ctx, dgst, content) + resIndex := res.(*manifestlist.DeserializedManifestList) + if err == nil && resIndex.Manifests != nil { + return resIndex, nil + } + + // Otherwise, assume it must be an image manifest + return ms.ocischemaHandler.Unmarshal(ctx, dgst, content) default: return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)} } From 132abc6de59b8f68f047be70b85edf5932e8f09f Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Thu, 15 Mar 2018 16:04:58 -0400 Subject: [PATCH 17/20] Test storing OCI image manifests and indexes with/without a media type OCI Image manifests and indexes are supported both with and without an embeded MediaType (the field is reserved according to the spec). Test storing and retrieving both types from the manifest store. Signed-off-by: Owen W. Taylor --- manifest/ocischema/builder.go | 30 ++++- registry/storage/manifeststore_test.go | 153 ++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 7 deletions(-) diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go index 9bfef614..22a53c0d 100644 --- a/manifest/ocischema/builder.go +++ b/manifest/ocischema/builder.go @@ -4,12 +4,13 @@ import ( "context" "github.com/docker/distribution" + "github.com/docker/distribution/manifest" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go/v1" ) // builder is a type for constructing manifests. -type builder struct { +type Builder struct { // bs is a BlobService used to publish the configuration blob. bs distribution.BlobService @@ -22,26 +23,43 @@ type builder struct { // Annotations contains arbitrary metadata relating to the targeted content. annotations map[string]string + + // For testing purposes + mediaType string } // NewManifestBuilder is used to build new manifests for the current schema // version. It takes a BlobService so it can publish the configuration blob // as part of the Build process, and annotations. func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotations map[string]string) distribution.ManifestBuilder { - mb := &builder{ + mb := &Builder{ bs: bs, configJSON: make([]byte, len(configJSON)), annotations: annotations, + mediaType: v1.MediaTypeImageManifest, } copy(mb.configJSON, configJSON) return mb } +// For testing purposes, we want to be able to create an OCI image with +// either an MediaType either empty, or with the OCI image value +func (mb *Builder) SetMediaType(mediaType string) { + if mediaType != "" && mediaType != v1.MediaTypeImageManifest { + panic("Invalid media type for OCI image manifest") + } + + mb.mediaType = mediaType +} + // Build produces a final manifest from the given references. -func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { +func (mb *Builder) Build(ctx context.Context) (distribution.Manifest, error) { m := Manifest{ - Versioned: SchemaVersion, + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: mb.mediaType, + }, Layers: make([]distribution.Descriptor, len(mb.layers)), Annotations: mb.annotations, } @@ -76,12 +94,12 @@ func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) { } // AppendReference adds a reference to the current ManifestBuilder. -func (mb *builder) AppendReference(d distribution.Describable) error { +func (mb *Builder) AppendReference(d distribution.Describable) error { mb.layers = append(mb.layers, d.Descriptor()) return nil } // References returns the current references added to this builder. -func (mb *builder) References() []distribution.Descriptor { +func (mb *Builder) References() []distribution.Descriptor { return mb.layers } diff --git a/registry/storage/manifeststore_test.go b/registry/storage/manifeststore_test.go index 2fad7611..61a4beee 100644 --- a/registry/storage/manifeststore_test.go +++ b/registry/storage/manifeststore_test.go @@ -9,6 +9,8 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/ocischema" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/reference" "github.com/docker/distribution/registry/storage/cache/memory" @@ -17,6 +19,7 @@ import ( "github.com/docker/distribution/testutil" "github.com/docker/libtrust" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" ) type manifestStoreTestEnv struct { @@ -356,6 +359,155 @@ func testManifestStorage(t *testing.T, options ...RegistryOption) { } } +func TestOCIManifestStorage(t *testing.T) { + testOCIManifestStorage(t, "includeMediaTypes=true", true) + testOCIManifestStorage(t, "includeMediaTypes=false", false) +} + +func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes bool) { + var imageMediaType string + var indexMediaType string + if includeMediaTypes { + imageMediaType = v1.MediaTypeImageManifest + indexMediaType = v1.MediaTypeImageIndex + } else { + imageMediaType = "" + indexMediaType = "" + } + + repoName, _ := reference.WithName("foo/bar") + env := newManifestStoreTestEnv(t, repoName, "thetag", + BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), + EnableDelete, EnableRedirect) + + ctx := context.Background() + ms, err := env.repository.Manifests(ctx) + if err != nil { + t.Fatal(err) + } + + // Build a manifest and store it and its layers in the registry + + blobStore := env.repository.Blobs(ctx) + builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{}) + builder.(*ocischema.Builder).SetMediaType(imageMediaType) + + // Add some layers + for i := 0; i < 2; i++ { + rs, ds, err := testutil.CreateRandomTarFile() + if err != nil { + t.Fatalf("%s: unexpected error generating test layer file", testname) + } + dgst := digest.Digest(ds) + + wr, err := env.repository.Blobs(env.ctx).Create(env.ctx) + if err != nil { + t.Fatalf("%s: unexpected error creating test upload: %v", testname, err) + } + + if _, err := io.Copy(wr, rs); err != nil { + t.Fatalf("%s: unexpected error copying to upload: %v", testname, err) + } + + if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil { + t.Fatalf("%s: unexpected error finishing upload: %v", testname, err) + } + + builder.AppendReference(distribution.Descriptor{Digest: dgst}) + } + + manifest, err := builder.Build(ctx) + if err != nil { + t.Fatalf("%s: unexpected error generating manifest: %v", testname, err) + } + + var manifestDigest digest.Digest + if manifestDigest, err = ms.Put(ctx, manifest); err != nil { + t.Fatalf("%s: unexpected error putting manifest: %v", testname, err) + } + + // Also create an image index that contains the manifest + + descriptor, err := env.registry.BlobStatter().Stat(ctx, manifestDigest) + if err != nil { + t.Fatalf("%s: unexpected error getting manifest descriptor", testname) + } + descriptor.MediaType = v1.MediaTypeImageManifest + + platformSpec := manifestlist.PlatformSpec{ + Architecture: "atari2600", + OS: "CP/M", + } + + manifestDescriptors := []manifestlist.ManifestDescriptor{ + manifestlist.ManifestDescriptor{ + Descriptor: descriptor, + Platform: platformSpec, + }, + } + + imageIndex, err := manifestlist.FromDescriptorsWithMediaType(manifestDescriptors, indexMediaType) + if err != nil { + t.Fatalf("%s: unexpected error creating image index: %v", testname, err) + } + + var indexDigest digest.Digest + if indexDigest, err = ms.Put(ctx, imageIndex); err != nil { + t.Fatalf("%s: unexpected error putting image index: %v", testname, err) + } + + // Now check that we can retrieve the manifest + + fromStore, err := ms.Get(ctx, manifestDigest) + if err != nil { + t.Fatalf("%s: unexpected error fetching manifest: %v", testname, err) + } + + fetchedManifest, ok := fromStore.(*ocischema.DeserializedManifest) + if !ok { + t.Fatalf("%s: unexpected type for fetched manifest", testname) + } + + if fetchedManifest.MediaType != imageMediaType { + t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedManifest.MediaType) + } + + payloadMediaType, _, err := fromStore.Payload() + if err != nil { + t.Fatalf("%s: error getting payload %v", testname, err) + } + + if payloadMediaType != v1.MediaTypeImageManifest { + t.Fatalf("%s: unexpected MediaType for manifest payload, %s", testname, payloadMediaType) + } + + // and the image index + + fromStore, err = ms.Get(ctx, indexDigest) + if err != nil { + t.Fatalf("%s: unexpected error fetching image index: %v", testname, err) + } + + fetchedIndex, ok := fromStore.(*manifestlist.DeserializedManifestList) + if !ok { + t.Fatalf("%s: unexpected type for fetched manifest", testname) + } + + if fetchedIndex.MediaType != indexMediaType { + t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedManifest.MediaType) + } + + payloadMediaType, _, err = fromStore.Payload() + if err != nil { + t.Fatalf("%s: error getting payload %v", testname, err) + } + + if payloadMediaType != v1.MediaTypeImageIndex { + t.Fatalf("%s: unexpected MediaType for index payload, %s", testname, payloadMediaType) + } + +} + // TestLinkPathFuncs ensures that the link path functions behavior are locked // down and implemented as expected. func TestLinkPathFuncs(t *testing.T) { @@ -387,5 +539,4 @@ func TestLinkPathFuncs(t *testing.T) { t.Fatalf("incorrect path returned: %q != %q", p, testcase.expected) } } - } From e8d7941ca6a90ada87caaa9affb67565309aeeba Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Tue, 19 Jun 2018 13:48:42 -0500 Subject: [PATCH 18/20] address lint and gofmt issues Signed-off-by: Mike Brown --- manifest/manifestlist/manifestlist.go | 2 +- manifest/ocischema/builder.go | 6 +++--- registry/storage/manifeststore_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/manifest/manifestlist/manifestlist.go b/manifest/manifestlist/manifestlist.go index a08f1e5a..f4e915ee 100644 --- a/manifest/manifestlist/manifestlist.go +++ b/manifest/manifestlist/manifestlist.go @@ -154,7 +154,7 @@ func FromDescriptors(descriptors []ManifestDescriptor) (*DeserializedManifestLis return FromDescriptorsWithMediaType(descriptors, mediaType) } -// For testing purposes, it's useful to be able to specify the media type explicitly +// FromDescriptorsWithMediaType is for testing purposes, it's useful to be able to specify the media type explicitly func FromDescriptorsWithMediaType(descriptors []ManifestDescriptor, mediaType string) (*DeserializedManifestList, error) { m := ManifestList{ Versioned: manifest.Versioned{ diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go index 22a53c0d..c189a16e 100644 --- a/manifest/ocischema/builder.go +++ b/manifest/ocischema/builder.go @@ -9,7 +9,7 @@ import ( "github.com/opencontainers/image-spec/specs-go/v1" ) -// builder is a type for constructing manifests. +// Builder is a type for constructing manifests. type Builder struct { // bs is a BlobService used to publish the configuration blob. bs distribution.BlobService @@ -43,8 +43,8 @@ func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotati return mb } -// For testing purposes, we want to be able to create an OCI image with -// either an MediaType either empty, or with the OCI image value +// SetMediaType assigns the passed mediatype or error if the mediatype is not a +// valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json" func (mb *Builder) SetMediaType(mediaType string) { if mediaType != "" && mediaType != v1.MediaTypeImageManifest { panic("Invalid media type for OCI image manifest") diff --git a/registry/storage/manifeststore_test.go b/registry/storage/manifeststore_test.go index 61a4beee..6e583c53 100644 --- a/registry/storage/manifeststore_test.go +++ b/registry/storage/manifeststore_test.go @@ -440,7 +440,7 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo } manifestDescriptors := []manifestlist.ManifestDescriptor{ - manifestlist.ManifestDescriptor{ + { Descriptor: descriptor, Platform: platformSpec, }, From 5f588fbf9b2eb3cef7a6c5876319a4ec88c43c87 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Thu, 19 Jul 2018 16:07:26 -0500 Subject: [PATCH 19/20] address review comment regarding panic use Signed-off-by: Mike Brown --- manifest/ocischema/builder.go | 6 ++++-- registry/storage/manifeststore_test.go | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/manifest/ocischema/builder.go b/manifest/ocischema/builder.go index c189a16e..023a8a0b 100644 --- a/manifest/ocischema/builder.go +++ b/manifest/ocischema/builder.go @@ -2,6 +2,7 @@ package ocischema import ( "context" + "errors" "github.com/docker/distribution" "github.com/docker/distribution/manifest" @@ -45,12 +46,13 @@ func NewManifestBuilder(bs distribution.BlobService, configJSON []byte, annotati // SetMediaType assigns the passed mediatype or error if the mediatype is not a // valid media type for oci image manifests currently: "" or "application/vnd.oci.image.manifest.v1+json" -func (mb *Builder) SetMediaType(mediaType string) { +func (mb *Builder) SetMediaType(mediaType string) error { if mediaType != "" && mediaType != v1.MediaTypeImageManifest { - panic("Invalid media type for OCI image manifest") + return errors.New("Invalid media type for OCI image manifest") } mb.mediaType = mediaType + return nil } // Build produces a final manifest from the given references. diff --git a/registry/storage/manifeststore_test.go b/registry/storage/manifeststore_test.go index 6e583c53..c6a2ba06 100644 --- a/registry/storage/manifeststore_test.go +++ b/registry/storage/manifeststore_test.go @@ -390,7 +390,10 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo blobStore := env.repository.Blobs(ctx) builder := ocischema.NewManifestBuilder(blobStore, []byte{}, map[string]string{}) - builder.(*ocischema.Builder).SetMediaType(imageMediaType) + err = builder.(*ocischema.Builder).SetMediaType(imageMediaType) + if err != nil { + t.Fatal(err) + } // Add some layers for i := 0; i < 2; i++ { From 20aecf1d7b8c975e22679bb5e5b6502c49c111b5 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Thu, 19 Jul 2018 19:41:31 -0500 Subject: [PATCH 20/20] added test for initial oci schema version Signed-off-by: Mike Brown --- registry/storage/manifeststore_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/registry/storage/manifeststore_test.go b/registry/storage/manifeststore_test.go index c6a2ba06..7c9b9edf 100644 --- a/registry/storage/manifeststore_test.go +++ b/registry/storage/manifeststore_test.go @@ -475,6 +475,10 @@ func testOCIManifestStorage(t *testing.T, testname string, includeMediaTypes boo t.Fatalf("%s: unexpected MediaType for result, %s", testname, fetchedManifest.MediaType) } + if fetchedManifest.SchemaVersion != ocischema.SchemaVersion.SchemaVersion { + t.Fatalf("%s: unexpected schema version for result, %d", testname, fetchedManifest.SchemaVersion) + } + payloadMediaType, _, err := fromStore.Payload() if err != nil { t.Fatalf("%s: error getting payload %v", testname, err)