Implementation of the Manifest Service API refactor.

Add a generic Manifest interface to represent manifests in the registry and
remove references to schema specific manifests.

Add a ManifestBuilder to construct Manifest objects. Concrete manifest builders
will exist for each manifest type and implementations will contain manifest
specific data used to build a manifest.

Remove Signatures() from Repository interface.

Signatures are relevant only to schema1 manifests.  Move access to the signature
store inside the schema1 manifestStore.  Add some API tests to verify
signature roundtripping.

schema1
-------

Change the way data is stored in schema1.Manifest to enable Payload() to be used
to return complete Manifest JSON from the HTTP handler without knowledge of the
schema1 protocol.

tags
----

Move tag functionality to a seperate TagService and update ManifestService
to use the new interfaces.  Implement a driver based tagService to be backward
compatible with the current tag service.

Add a proxyTagService to enable the registry to get a digest for remote manifests
from a tag.

manifest store
--------------

Remove revision store and move all signing functionality into the signed manifeststore.

manifest registration
---------------------

Add a mechanism to register manifest media types and to allow different manifest
types to be Unmarshalled correctly.

client
------

Add ManifestServiceOptions to client functions to allow tags to be passed into Put and
Get for building correct registry URLs.  Change functional arguments to be an interface type
to allow passing data without mutating shared state.

Signed-off-by: Richard Scothern <richard.scothern@gmail.com>

Signed-off-by: Richard Scothern <richard.scothern@docker.com>

manifest/schema1/builder.go

@@ -0,0 +1,91 @@
+package schema1
+
+import (
+	"fmt"
+
+	"errors"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+	"github.com/docker/libtrust"
+)
+
+// ManifestBuilder is a type for constructing manifests
+type manifestBuilder struct {
+	Manifest
+	pk libtrust.PrivateKey
+}
+
+// NewManifestBuilder is used to build new manifests for the current schema
+// version.
+func NewManifestBuilder(pk libtrust.PrivateKey, name, tag, architecture string) distribution.ManifestBuilder {
+	return &manifestBuilder{
+		Manifest: Manifest{
+			Versioned: manifest.Versioned{
+				SchemaVersion: 1,
+			},
+			Name:         name,
+			Tag:          tag,
+			Architecture: architecture,
+		},
+		pk: pk,
+	}
+}
+
+// Build produces a final manifest from the given references
+func (mb *manifestBuilder) Build() (distribution.Manifest, error) {
+	m := mb.Manifest
+	if len(m.FSLayers) == 0 {
+		return nil, errors.New("cannot build manifest with zero layers or history")
+	}
+
+	m.FSLayers = make([]FSLayer, len(mb.Manifest.FSLayers))
+	m.History = make([]History, len(mb.Manifest.History))
+	copy(m.FSLayers, mb.Manifest.FSLayers)
+	copy(m.History, mb.Manifest.History)
+
+	return Sign(&m, mb.pk)
+}
+
+// AppendReference adds a reference to the current manifestBuilder
+func (mb *manifestBuilder) AppendReference(d distribution.Describable) error {
+	r, ok := d.(Reference)
+	if !ok {
+		return fmt.Errorf("Unable to add non-reference type to v1 builder")
+	}
+
+	// Entries need to be prepended
+	mb.Manifest.FSLayers = append([]FSLayer{{BlobSum: r.Digest}}, mb.Manifest.FSLayers...)
+	mb.Manifest.History = append([]History{r.History}, mb.Manifest.History...)
+	return nil
+
+}
+
+// References returns the current references added to this builder
+func (mb *manifestBuilder) References() []distribution.Descriptor {
+	refs := make([]distribution.Descriptor, len(mb.Manifest.FSLayers))
+	for i := range mb.Manifest.FSLayers {
+		layerDigest := mb.Manifest.FSLayers[i].BlobSum
+		history := mb.Manifest.History[i]
+		ref := Reference{layerDigest, 0, history}
+		refs[i] = ref.Descriptor()
+	}
+	return refs
+}
+
+// Reference describes a manifest v2, schema version 1 dependency.
+// An FSLayer associated with a history entry.
+type Reference struct {
+	Digest  digest.Digest
+	Size    int64 // if we know it, set it for the descriptor.
+	History History
+}
+
+// Descriptor describes a reference
+func (r Reference) Descriptor() distribution.Descriptor {
+	return distribution.Descriptor{
+		MediaType: MediaTypeManifestLayer,
+		Digest:    r.Digest,
+		Size:      r.Size,
+	}
+}

manifest/schema1/builder_test.go

@@ -0,0 +1,97 @@
+package schema1
+
+import (
+	"testing"
+
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+	"github.com/docker/libtrust"
+)
+
+func makeSignedManifest(t *testing.T, pk libtrust.PrivateKey, refs []Reference) *SignedManifest {
+	u := &Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 1,
+		},
+		Name:         "foo/bar",
+		Tag:          "latest",
+		Architecture: "amd64",
+	}
+
+	for i := len(refs) - 1; i >= 0; i-- {
+		u.FSLayers = append(u.FSLayers, FSLayer{
+			BlobSum: refs[i].Digest,
+		})
+		u.History = append(u.History, History{
+			V1Compatibility: refs[i].History.V1Compatibility,
+		})
+	}
+
+	signedManifest, err := Sign(u, pk)
+	if err != nil {
+		t.Fatalf("unexpected error signing manifest: %v", err)
+	}
+	return signedManifest
+}
+
+func TestBuilder(t *testing.T) {
+	pk, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatalf("unexpected error generating private key: %v", err)
+	}
+
+	r1 := Reference{
+		Digest:  "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		Size:    1,
+		History: History{V1Compatibility: "{\"a\" : 1 }"},
+	}
+	r2 := Reference{
+		Digest:  "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
+		Size:    2,
+		History: History{V1Compatibility: "{\"\a\" : 2 }"},
+	}
+
+	handCrafted := makeSignedManifest(t, pk, []Reference{r1, r2})
+
+	b := NewManifestBuilder(pk, handCrafted.Manifest.Name, handCrafted.Manifest.Tag, handCrafted.Manifest.Architecture)
+	_, err = b.Build()
+	if err == nil {
+		t.Fatal("Expected error building zero length manifest")
+	}
+
+	err = b.AppendReference(r1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.AppendReference(r2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	refs := b.References()
+	if len(refs) != 2 {
+		t.Fatalf("Unexpected reference count : %d != %d", 2, len(refs))
+	}
+
+	// Ensure ordering
+	if refs[0].Digest != r2.Digest {
+		t.Fatalf("Unexpected reference : %v", refs[0])
+	}
+
+	m, err := b.Build()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	built, ok := m.(*SignedManifest)
+	if !ok {
+		t.Fatalf("unexpected type from Build() : %T", built)
+	}
+
+	d1 := digest.FromBytes(built.Canonical)
+	d2 := digest.FromBytes(handCrafted.Canonical)
+	if d1 != d2 {
+		t.Errorf("mismatching canonical JSON")
+	}
+}

manifest/schema1/manifest.go

@@ -2,20 +2,22 @@ package schema1
 
 import (
 	"encoding/json"
+	"fmt"
 
+	"github.com/docker/distribution"
 	"github.com/docker/distribution/digest"
 	"github.com/docker/distribution/manifest"
 	"github.com/docker/libtrust"
 )
 
-// TODO(stevvooe): When we rev the manifest format, the contents of this
-// package should be moved to manifest/v1.
-
 const (
-	// ManifestMediaType specifies the mediaType for the current version. Note
-	// that for schema version 1, the the media is optionally
-	// "application/json".
-	ManifestMediaType = "application/vnd.docker.distribution.manifest.v1+json"
+	// MediaTypeManifest specifies the mediaType for the current version. Note
+	// that for schema version 1, the the media is optionally "application/json".
+	MediaTypeManifest = "application/vnd.docker.distribution.manifest.v1+json"
+	// MediaTypeSignedManifest specifies the mediatype for current SignedManifest version
+	MediaTypeSignedManifest = "application/vnd.docker.distribution.manifest.v1+prettyjws"
+	// MediaTypeManifestLayer specifies the media type for manifest layers
+	MediaTypeManifestLayer = "application/vnd.docker.container.image.rootfs.diff+x-gtar"
 )
 
 var (
@@ -26,6 +28,47 @@ var (
 	}
 )
 
+func init() {
+	schema1Func := func(b []byte) (distribution.Manifest, distribution.Descriptor, error) {
+		sm := new(SignedManifest)
+		err := sm.UnmarshalJSON(b)
+		if err != nil {
+			return nil, distribution.Descriptor{}, err
+		}
+
+		desc := distribution.Descriptor{
+			Digest:    digest.FromBytes(sm.Canonical),
+			Size:      int64(len(sm.Canonical)),
+			MediaType: MediaTypeManifest,
+		}
+		return sm, desc, err
+	}
+	err := distribution.RegisterManifestSchema(MediaTypeManifest, schema1Func)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
+	}
+	err = distribution.RegisterManifestSchema("", schema1Func)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
+	}
+	err = distribution.RegisterManifestSchema("application/json; charset=utf-8", schema1Func)
+	if err != nil {
+		panic(fmt.Sprintf("Unable to register manifest: %s", err))
+	}
+}
+
+// FSLayer is a container struct for BlobSums defined in an image manifest
+type FSLayer struct {
+	// BlobSum is the tarsum of the referenced filesystem image layer
+	BlobSum digest.Digest `json:"blobSum"`
+}
+
+// History stores unstructured v1 compatibility information
+type History struct {
+	// V1Compatibility is the raw v1 compatibility information
+	V1Compatibility string `json:"v1Compatibility"`
+}
+
 // Manifest provides the base accessible fields for working with V2 image
 // format in the registry.
 type Manifest struct {
@@ -49,59 +92,64 @@ type Manifest struct {
 }
 
 // SignedManifest provides an envelope for a signed image manifest, including
-// the format sensitive raw bytes. It contains fields to
+// the format sensitive raw bytes.
 type SignedManifest struct {
 	Manifest
 
-	// Raw is the byte representation of the ImageManifest, used for signature
-	// verification. The value of Raw must be used directly during
-	// serialization, or the signature check will fail. The manifest byte
+	// Canonical is the canonical byte representation of the ImageManifest,
+	// without any attached signatures. The manifest byte
 	// representation cannot change or it will have to be re-signed.
-	Raw []byte `json:"-"`
+	Canonical []byte `json:"-"`
+
+	// all contains the byte representation of the Manifest including signatures
+	// and is retuend by Payload()
+	all []byte
 }
 
-// UnmarshalJSON populates a new ImageManifest struct from JSON data.
+// UnmarshalJSON populates a new SignedManifest struct from JSON data.
 func (sm *SignedManifest) UnmarshalJSON(b []byte) error {
-	sm.Raw = make([]byte, len(b), len(b))
-	copy(sm.Raw, b)
+	sm.all = make([]byte, len(b), len(b))
+	// store manifest and signatures in all
+	copy(sm.all, b)
 
-	p, err := sm.Payload()
+	jsig, err := libtrust.ParsePrettySignature(b, "signatures")
 	if err != nil {
 		return err
 	}
 
+	// Resolve the payload in the manifest.
+	bytes, err := jsig.Payload()
+	if err != nil {
+		return err
+	}
+
+	// sm.Canonical stores the canonical manifest JSON
+	sm.Canonical = make([]byte, len(bytes), len(bytes))
+	copy(sm.Canonical, bytes)
+
+	// Unmarshal canonical JSON into Manifest object
 	var manifest Manifest
-	if err := json.Unmarshal(p, &manifest); err != nil {
+	if err := json.Unmarshal(sm.Canonical, &manifest); err != nil {
 		return err
 	}
 
 	sm.Manifest = manifest
+
 	return nil
 }
 
-// Payload returns the raw, signed content of the signed manifest. The
-// contents can be used to calculate the content identifier.
-func (sm *SignedManifest) Payload() ([]byte, error) {
-	jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
-	if err != nil {
-		return nil, err
+// References returnes the descriptors of this manifests references
+func (sm SignedManifest) References() []distribution.Descriptor {
+	dependencies := make([]distribution.Descriptor, len(sm.FSLayers))
+	for i, fsLayer := range sm.FSLayers {
+		dependencies[i] = distribution.Descriptor{
+			MediaType: "application/vnd.docker.container.image.rootfs.diff+x-gtar",
+			Digest:    fsLayer.BlobSum,
+		}
 	}
 
-	// Resolve the payload in the manifest.
-	return jsig.Payload()
-}
+	return dependencies
 
-// Signatures returns the signatures as provided by
-// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws
-// signatures.
-func (sm *SignedManifest) Signatures() ([][]byte, error) {
-	jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
-	if err != nil {
-		return nil, err
-	}
-
-	// Resolve the payload in the manifest.
-	return jsig.Signatures()
 }
 
 // MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner
@@ -109,22 +157,28 @@ func (sm *SignedManifest) Signatures() ([][]byte, error) {
 // use Raw directly, since the the content produced by json.Marshal will be
 // compacted and will fail signature checks.
 func (sm *SignedManifest) MarshalJSON() ([]byte, error) {
-	if len(sm.Raw) > 0 {
-		return sm.Raw, nil
+	if len(sm.all) > 0 {
+		return sm.all, nil
 	}
 
 	// If the raw data is not available, just dump the inner content.
 	return json.Marshal(&sm.Manifest)
 }
 
-// FSLayer is a container struct for BlobSums defined in an image manifest
-type FSLayer struct {
-	// BlobSum is the digest of the referenced filesystem image layer
-	BlobSum digest.Digest `json:"blobSum"`
+// Payload returns the signed content of the signed manifest.
+func (sm SignedManifest) Payload() (string, []byte, error) {
+	return MediaTypeManifest, sm.all, nil
 }
 
-// History stores unstructured v1 compatibility information
-type History struct {
-	// V1Compatibility is the raw v1 compatibility information
-	V1Compatibility string `json:"v1Compatibility"`
+// Signatures returns the signatures as provided by
+// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws
+// signatures.
+func (sm *SignedManifest) Signatures() ([][]byte, error) {
+	jsig, err := libtrust.ParsePrettySignature(sm.all, "signatures")
+	if err != nil {
+		return nil, err
+	}
+
+	// Resolve the payload in the manifest.
+	return jsig.Signatures()
 }

manifest/schema1/manifest_test.go

@@ -19,15 +19,15 @@ type testEnv struct {
 func TestManifestMarshaling(t *testing.T) {
 	env := genEnv(t)
 
-	// Check that the Raw field is the same as json.MarshalIndent with these
+	// Check that the all field is the same as json.MarshalIndent with these
 	// parameters.
 	p, err := json.MarshalIndent(env.signed, "", "   ")
 	if err != nil {
 		t.Fatalf("error marshaling manifest: %v", err)
 	}
 
-	if !bytes.Equal(p, env.signed.Raw) {
-		t.Fatalf("manifest bytes not equal: %q != %q", string(env.signed.Raw), string(p))
+	if !bytes.Equal(p, env.signed.all) {
+		t.Fatalf("manifest bytes not equal: %q != %q", string(env.signed.all), string(p))
 	}
 }
 
@@ -35,7 +35,7 @@ func TestManifestUnmarshaling(t *testing.T) {
 	env := genEnv(t)
 
 	var signed SignedManifest
-	if err := json.Unmarshal(env.signed.Raw, &signed); err != nil {
+	if err := json.Unmarshal(env.signed.all, &signed); err != nil {
 		t.Fatalf("error unmarshaling signed manifest: %v", err)
 	}
 

manifest/schema1/sign.go

@@ -31,8 +31,9 @@ func Sign(m *Manifest, pk libtrust.PrivateKey) (*SignedManifest, error) {
 	}
 
 	return &SignedManifest{
-		Manifest: *m,
-		Raw:      pretty,
+		Manifest:  *m,
+		all:       pretty,
+		Canonical: p,
 	}, nil
 }
 
@@ -60,7 +61,8 @@ func SignWithChain(m *Manifest, key libtrust.PrivateKey, chain []*x509.Certifica
 	}
 
 	return &SignedManifest{
-		Manifest: *m,
-		Raw:      pretty,
+		Manifest:  *m,
+		all:       pretty,
+		Canonical: p,
 	}, nil
 }

manifest/schema1/verify.go

@@ -10,7 +10,7 @@ import (
 // Verify verifies the signature of the signed manifest returning the public
 // keys used during signing.
 func Verify(sm *SignedManifest) ([]libtrust.PublicKey, error) {
-	js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
+	js, err := libtrust.ParsePrettySignature(sm.all, "signatures")
 	if err != nil {
 		logrus.WithField("err", err).Debugf("(*SignedManifest).Verify")
 		return nil, err
@@ -23,7 +23,7 @@ func Verify(sm *SignedManifest) ([]libtrust.PublicKey, error) {
 // certificate pool returning the list of verified chains. Signatures without
 // an x509 chain are not checked.
 func VerifyChains(sm *SignedManifest, ca *x509.CertPool) ([][]*x509.Certificate, error) {
-	js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
+	js, err := libtrust.ParsePrettySignature(sm.all, "signatures")
 	if err != nil {
 		return nil, err
 	}