Add schema2 manifest support
Add schema2 manifest implementation. Add a schema2 builder that creates a schema2 manifest from descriptors and a configuration. It will add the configuration to the blob store if necessary. Rename the original schema1 manifest builder to ReferenceBuilder, and create a ConfigBuilder variant that can build a schema1 manifest from an image configuration and set of descriptors. This will be used to translate schema2 manifests to the schema1 format for backward compatibliity, by adding the descriptors from the existing schema2 manifest to the schema1 builder. It will also be used by engine-side push code to create schema1 manifests from the new-style image configration, when necessary to push a schema1 manifest. Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>
This commit is contained in:
Aaron Lehmann
parent
83ccbd18c0
commit
2ff77c00ba
10 changed files with 1088 additions and 16 deletions
78
manifest/schema2/builder.go
Normal file
78
manifest/schema2/builder.go
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
package schema2
|
||||
|
||||
import (
|
||||
"github.com/docker/distribution"
|
||||
"github.com/docker/distribution/context"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/manifest"
|
||||
)
|
||||
|
||||
// builder is a type for constructing manifests.
|
||||
type builder struct {
|
||||
// bs is a BlobService used to publish the configuration blob.
|
||||
bs distribution.BlobService
|
||||
|
||||
// configJSON references
|
||||
configJSON []byte
|
||||
|
||||
// layers is a list of layer descriptors that gets built by successive
|
||||
// calls to AppendReference.
|
||||
layers []distribution.Descriptor
|
||||
}
|
||||
|
||||
// NewManifestBuilder is used to build new manifests for the current schema
|
||||
// version. It takes a BlobService so it can publish the configuration blob
|
||||
// as part of the Build process.
|
||||
func NewManifestBuilder(bs distribution.BlobService, configJSON []byte) distribution.ManifestBuilder {
|
||||
mb := &builder{
|
||||
bs: bs,
|
||||
configJSON: make([]byte, len(configJSON)),
|
||||
}
|
||||
copy(mb.configJSON, configJSON)
|
||||
|
||||
return mb
|
||||
}
|
||||
|
||||
// Build produces a final manifest from the given references.
|
||||
func (mb *builder) Build(ctx context.Context) (distribution.Manifest, error) {
|
||||
m := Manifest{
|
||||
Versioned: manifest.Versioned{
|
||||
SchemaVersion: 2,
|
||||
},
|
||||
MediaType: MediaTypeManifest,
|
||||
Layers: make([]distribution.Descriptor, len(mb.layers)),
|
||||
}
|
||||
copy(m.Layers, mb.layers)
|
||||
|
||||
configDigest := digest.FromBytes(mb.configJSON)
|
||||
|
||||
var err error
|
||||
m.Config, err = mb.bs.Stat(ctx, configDigest)
|
||||
switch err {
|
||||
case nil:
|
||||
return FromStruct(m)
|
||||
case distribution.ErrBlobUnknown:
|
||||
// nop
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add config to the blob store
|
||||
m.Config, err = mb.bs.Put(ctx, MediaTypeConfig, mb.configJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return FromStruct(m)
|
||||
}
|
||||
|
||||
// AppendReference adds a reference to the current ManifestBuilder.
|
||||
func (mb *builder) AppendReference(d distribution.Describable) error {
|
||||
mb.layers = append(mb.layers, d.Descriptor())
|
||||
return nil
|
||||
}
|
||||
|
||||
// References returns the current references added to this builder.
|
||||
func (mb *builder) References() []distribution.Descriptor {
|
||||
return mb.layers
|
||||
}
|
||||
210
manifest/schema2/builder_test.go
Normal file
210
manifest/schema2/builder_test.go
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
package schema2
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution"
|
||||
"github.com/docker/distribution/context"
|
||||
"github.com/docker/distribution/digest"
|
||||
)
|
||||
|
||||
type mockBlobService struct {
|
||||
descriptors map[digest.Digest]distribution.Descriptor
|
||||
}
|
||||
|
||||
func (bs *mockBlobService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
|
||||
if descriptor, ok := bs.descriptors[dgst]; ok {
|
||||
return descriptor, nil
|
||||
}
|
||||
return distribution.Descriptor{}, distribution.ErrBlobUnknown
|
||||
}
|
||||
|
||||
func (bs *mockBlobService) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func (bs *mockBlobService) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func (bs *mockBlobService) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
|
||||
d := distribution.Descriptor{
|
||||
Digest: digest.FromBytes(p),
|
||||
Size: int64(len(p)),
|
||||
MediaType: mediaType,
|
||||
}
|
||||
bs.descriptors[d.Digest] = d
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (bs *mockBlobService) Create(ctx context.Context) (distribution.BlobWriter, error) {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func (bs *mockBlobService) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
|
||||
panic("not implemented")
|
||||
}
|
||||
|
||||
func TestBuilder(t *testing.T) {
|
||||
imgJSON := []byte(`{
|
||||
"architecture": "amd64",
|
||||
"config": {
|
||||
"AttachStderr": false,
|
||||
"AttachStdin": false,
|
||||
"AttachStdout": false,
|
||||
"Cmd": [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
"echo hi"
|
||||
],
|
||||
"Domainname": "",
|
||||
"Entrypoint": null,
|
||||
"Env": [
|
||||
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||
"derived=true",
|
||||
"asdf=true"
|
||||
],
|
||||
"Hostname": "23304fc829f9",
|
||||
"Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
|
||||
"Labels": {},
|
||||
"OnBuild": [],
|
||||
"OpenStdin": false,
|
||||
"StdinOnce": false,
|
||||
"Tty": false,
|
||||
"User": "",
|
||||
"Volumes": null,
|
||||
"WorkingDir": ""
|
||||
},
|
||||
"container": "e91032eb0403a61bfe085ff5a5a48e3659e5a6deae9f4d678daa2ae399d5a001",
|
||||
"container_config": {
|
||||
"AttachStderr": false,
|
||||
"AttachStdin": false,
|
||||
"AttachStdout": false,
|
||||
"Cmd": [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
"#(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]"
|
||||
],
|
||||
"Domainname": "",
|
||||
"Entrypoint": null,
|
||||
"Env": [
|
||||
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||
"derived=true",
|
||||
"asdf=true"
|
||||
],
|
||||
"Hostname": "23304fc829f9",
|
||||
"Image": "sha256:4ab15c48b859c2920dd5224f92aabcd39a52794c5b3cf088fb3bbb438756c246",
|
||||
"Labels": {},
|
||||
"OnBuild": [],
|
||||
"OpenStdin": false,
|
||||
"StdinOnce": false,
|
||||
"Tty": false,
|
||||
"User": "",
|
||||
"Volumes": null,
|
||||
"WorkingDir": ""
|
||||
},
|
||||
"created": "2015-11-04T23:06:32.365666163Z",
|
||||
"docker_version": "1.9.0-dev",
|
||||
"history": [
|
||||
{
|
||||
"created": "2015-10-31T22:22:54.690851953Z",
|
||||
"created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /"
|
||||
},
|
||||
{
|
||||
"created": "2015-10-31T22:22:55.613815829Z",
|
||||
"created_by": "/bin/sh -c #(nop) CMD [\"sh\"]"
|
||||
},
|
||||
{
|
||||
"created": "2015-11-04T23:06:30.934316144Z",
|
||||
"created_by": "/bin/sh -c #(nop) ENV derived=true",
|
||||
"empty_layer": true
|
||||
},
|
||||
{
|
||||
"created": "2015-11-04T23:06:31.192097572Z",
|
||||
"created_by": "/bin/sh -c #(nop) ENV asdf=true",
|
||||
"empty_layer": true
|
||||
},
|
||||
{
|
||||
"created": "2015-11-04T23:06:32.083868454Z",
|
||||
"created_by": "/bin/sh -c dd if=/dev/zero of=/file bs=1024 count=1024"
|
||||
},
|
||||
{
|
||||
"created": "2015-11-04T23:06:32.365666163Z",
|
||||
"created_by": "/bin/sh -c #(nop) CMD [\"/bin/sh\" \"-c\" \"echo hi\"]",
|
||||
"empty_layer": true
|
||||
}
|
||||
],
|
||||
"os": "linux",
|
||||
"rootfs": {
|
||||
"diff_ids": [
|
||||
"sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
|
||||
"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
|
||||
"sha256:13f53e08df5a220ab6d13c58b2bf83a59cbdc2e04d0a3f041ddf4b0ba4112d49"
|
||||
],
|
||||
"type": "layers"
|
||||
}
|
||||
}`)
|
||||
configDigest := digest.FromBytes(imgJSON)
|
||||
|
||||
descriptors := []distribution.Descriptor{
|
||||
{
|
||||
Digest: digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
|
||||
Size: 5312,
|
||||
MediaType: MediaTypeLayer,
|
||||
},
|
||||
{
|
||||
Digest: digest.Digest("sha256:86e0e091d0da6bde2456dbb48306f3956bbeb2eae1b5b9a43045843f69fe4aaa"),
|
||||
Size: 235231,
|
||||
MediaType: MediaTypeLayer,
|
||||
},
|
||||
{
|
||||
Digest: digest.Digest("sha256:b4ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"),
|
||||
Size: 639152,
|
||||
MediaType: MediaTypeLayer,
|
||||
},
|
||||
}
|
||||
|
||||
bs := &mockBlobService{descriptors: make(map[digest.Digest]distribution.Descriptor)}
|
||||
builder := NewManifestBuilder(bs, imgJSON)
|
||||
|
||||
for _, d := range descriptors {
|
||||
if err := builder.AppendReference(d); err != nil {
|
||||
t.Fatalf("AppendReference returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
built, err := builder.Build(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Build returned error: %v", err)
|
||||
}
|
||||
|
||||
// Check that the config was put in the blob store
|
||||
_, err = bs.Stat(context.Background(), configDigest)
|
||||
if err != nil {
|
||||
t.Fatal("config was not put in the blob store")
|
||||
}
|
||||
|
||||
manifest := built.(*DeserializedManifest).Manifest
|
||||
|
||||
if manifest.Versioned.SchemaVersion != 2 {
|
||||
t.Fatal("SchemaVersion != 2")
|
||||
}
|
||||
|
||||
target := manifest.Target()
|
||||
if target.Digest != configDigest {
|
||||
t.Fatalf("unexpected digest in target: %s", target.Digest.String())
|
||||
}
|
||||
if target.MediaType != MediaTypeConfig {
|
||||
t.Fatalf("unexpected media type in target: %s", target.MediaType)
|
||||
}
|
||||
if target.Size != 3153 {
|
||||
t.Fatalf("unexpected size in target: %d", target.Size)
|
||||
}
|
||||
|
||||
references := manifest.References()
|
||||
|
||||
if !reflect.DeepEqual(references, descriptors) {
|
||||
t.Fatal("References() does not match the descriptors added")
|
||||
}
|
||||
}
|
||||
125
manifest/schema2/manifest.go
Normal file
125
manifest/schema2/manifest.go
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
package schema2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/docker/distribution"
|
||||
"github.com/docker/distribution/digest"
|
||||
"github.com/docker/distribution/manifest"
|
||||
)
|
||||
|
||||
const (
|
||||
// MediaTypeManifest specifies the mediaType for the current version.
|
||||
MediaTypeManifest = "application/vnd.docker.distribution.manifest.v2+json"
|
||||
|
||||
// MediaTypeConfig specifies the mediaType for the image configuration.
|
||||
MediaTypeConfig = "application/vnd.docker.container.image.v1+json"
|
||||
|
||||
// MediaTypeLayer is the mediaType used for layers referenced by the
|
||||
// manifest.
|
||||
MediaTypeLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip"
|
||||
)
|
||||
|
||||
var (
|
||||
// SchemaVersion provides a pre-initialized version structure for this
|
||||
// packages version of the manifest.
|
||||
SchemaVersion = manifest.Versioned{
|
||||
SchemaVersion: 2,
|
||||
}
|
||||
)
|
||||
|
||||
func init() {
|
||||
schema2Func := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
|
||||
m := new(DeserializedManifest)
|
||||
err := m.UnmarshalJSON(b)
|
||||
if err != nil {
|
||||
return nil, distribution.Descriptor{}, err
|
||||
}
|
||||
|
||||
dgst := digest.FromBytes(b)
|
||||
return m, distribution.Descriptor{Digest: dgst, Size: int64(len(b)), MediaType: MediaTypeManifest}, err
|
||||
}
|
||||
err := distribution.RegisterManifestSchema(MediaTypeManifest, schema2Func)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Unable to register manifest: %s", err))
|
||||
}
|
||||
}
|
||||
|
||||
// Manifest defines a schema2 manifest.
|
||||
type Manifest struct {
|
||||
manifest.Versioned
|
||||
MediaType string `json:"mediaType"`
|
||||
|
||||
// Config references the image configuration as a blob.
|
||||
Config distribution.Descriptor `json:"config"`
|
||||
|
||||
// Layers lists descriptors for the layers referenced by the
|
||||
// configuration.
|
||||
Layers []distribution.Descriptor `json:"layers"`
|
||||
}
|
||||
|
||||
// References returnes the descriptors of this manifests references.
|
||||
func (m Manifest) References() []distribution.Descriptor {
|
||||
return m.Layers
|
||||
|
||||
}
|
||||
|
||||
// Target returns the target of this signed manifest.
|
||||
func (m Manifest) Target() distribution.Descriptor {
|
||||
return m.Config
|
||||
}
|
||||
|
||||
// DeserializedManifest wraps Manifest with a copy of the original JSON.
|
||||
// It satisfies the distribution.Manifest interface.
|
||||
type DeserializedManifest struct {
|
||||
Manifest
|
||||
|
||||
// canonical is the canonical byte representation of the Manifest.
|
||||
canonical []byte
|
||||
}
|
||||
|
||||
// FromStruct takes a Manifest structure, marshals it to JSON, and returns a
|
||||
// DeserializedManifest which contains the manifest and its JSON representation.
|
||||
func FromStruct(m Manifest) (*DeserializedManifest, error) {
|
||||
var deserialized DeserializedManifest
|
||||
deserialized.Manifest = m
|
||||
|
||||
var err error
|
||||
deserialized.canonical, err = json.MarshalIndent(&m, "", " ")
|
||||
return &deserialized, err
|
||||
}
|
||||
|
||||
// UnmarshalJSON populates a new Manifest struct from JSON data.
|
||||
func (m *DeserializedManifest) UnmarshalJSON(b []byte) error {
|
||||
m.canonical = make([]byte, len(b), len(b))
|
||||
// store manifest in canonical
|
||||
copy(m.canonical, b)
|
||||
|
||||
// Unmarshal canonical JSON into Manifest object
|
||||
var manifest Manifest
|
||||
if err := json.Unmarshal(m.canonical, &manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.Manifest = manifest
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarshalJSON returns the contents of canonical. If canonical is empty,
|
||||
// marshals the inner contents.
|
||||
func (m *DeserializedManifest) MarshalJSON() ([]byte, error) {
|
||||
if len(m.canonical) > 0 {
|
||||
return m.canonical, nil
|
||||
}
|
||||
|
||||
return nil, errors.New("JSON representation not initialized in DeserializedManifest")
|
||||
}
|
||||
|
||||
// Payload returns the raw content of the manifest. The contents can be used to
|
||||
// calculate the content identifier.
|
||||
func (m DeserializedManifest) Payload() (string, []byte, error) {
|
||||
return m.MediaType, m.canonical, nil
|
||||
}
|
||||
106
manifest/schema2/manifest_test.go
Normal file
106
manifest/schema2/manifest_test.go
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
package schema2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/distribution"
|
||||
)
|
||||
|
||||
var expectedManifestSerialization = []byte(`{
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.docker.container.image.v1+json",
|
||||
"size": 985,
|
||||
"digest": "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b"
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": 153263,
|
||||
"digest": "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b"
|
||||
}
|
||||
]
|
||||
}`)
|
||||
|
||||
func TestManifest(t *testing.T) {
|
||||
manifest := Manifest{
|
||||
Versioned: SchemaVersion,
|
||||
MediaType: MediaTypeManifest,
|
||||
Config: distribution.Descriptor{
|
||||
Digest: "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b",
|
||||
Size: 985,
|
||||
MediaType: MediaTypeConfig,
|
||||
},
|
||||
Layers: []distribution.Descriptor{
|
||||
{
|
||||
Digest: "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b",
|
||||
Size: 153263,
|
||||
MediaType: MediaTypeLayer,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
deserialized, err := FromStruct(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("error creating DeserializedManifest: %v", err)
|
||||
}
|
||||
|
||||
mediaType, canonical, err := deserialized.Payload()
|
||||
|
||||
if mediaType != MediaTypeManifest {
|
||||
t.Fatalf("unexpected media type: %s", mediaType)
|
||||
}
|
||||
|
||||
// Check that the canonical field is the same as json.MarshalIndent
|
||||
// with these parameters.
|
||||
p, err := json.MarshalIndent(&manifest, "", " ")
|
||||
if err != nil {
|
||||
t.Fatalf("error marshaling manifest: %v", err)
|
||||
}
|
||||
if !bytes.Equal(p, canonical) {
|
||||
t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(p))
|
||||
}
|
||||
|
||||
// Check that canonical field matches expected value.
|
||||
if !bytes.Equal(expectedManifestSerialization, canonical) {
|
||||
t.Fatalf("manifest bytes not equal: %q != %q", string(canonical), string(expectedManifestSerialization))
|
||||
}
|
||||
|
||||
var unmarshalled DeserializedManifest
|
||||
if err := json.Unmarshal(deserialized.canonical, &unmarshalled); err != nil {
|
||||
t.Fatalf("error unmarshaling manifest: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(&unmarshalled, deserialized) {
|
||||
t.Fatalf("manifests are different after unmarshaling: %v != %v", unmarshalled, *deserialized)
|
||||
}
|
||||
|
||||
target := deserialized.Target()
|
||||
if target.Digest != "sha256:1a9ec845ee94c202b2d5da74a24f0ed2058318bfa9879fa541efaecba272e86b" {
|
||||
t.Fatalf("unexpected digest in target: %s", target.Digest.String())
|
||||
}
|
||||
if target.MediaType != MediaTypeConfig {
|
||||
t.Fatalf("unexpected media type in target: %s", target.MediaType)
|
||||
}
|
||||
if target.Size != 985 {
|
||||
t.Fatalf("unexpected size in target: %d", target.Size)
|
||||
}
|
||||
|
||||
references := deserialized.References()
|
||||
if len(references) != 1 {
|
||||
t.Fatalf("unexpected number of references: %d", len(references))
|
||||
}
|
||||
if references[0].Digest != "sha256:62d8908bee94c202b2d35224a221aaa2058318bfa9879fa541efaecba272331b" {
|
||||
t.Fatalf("unexpected digest in reference: %s", references[0].Digest.String())
|
||||
}
|
||||
if references[0].MediaType != MediaTypeLayer {
|
||||
t.Fatalf("unexpected media type in reference: %s", references[0].MediaType)
|
||||
}
|
||||
if references[0].Size != 153263 {
|
||||
t.Fatalf("unexpected size in reference: %d", references[0].Size)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue