vbatts/registry
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Implementation of the Manifest Service API refactor.

Browse source 
Add a generic Manifest interface to represent manifests in the registry and
remove references to schema specific manifests.

Add a ManifestBuilder to construct Manifest objects. Concrete manifest builders
will exist for each manifest type and implementations will contain manifest
specific data used to build a manifest.

Remove Signatures() from Repository interface.

Signatures are relevant only to schema1 manifests.  Move access to the signature
store inside the schema1 manifestStore.  Add some API tests to verify
signature roundtripping.

schema1
-------

Change the way data is stored in schema1.Manifest to enable Payload() to be used
to return complete Manifest JSON from the HTTP handler without knowledge of the
schema1 protocol.

tags
----

Move tag functionality to a seperate TagService and update ManifestService
to use the new interfaces.  Implement a driver based tagService to be backward
compatible with the current tag service.

Add a proxyTagService to enable the registry to get a digest for remote manifests
from a tag.

manifest store
--------------

Remove revision store and move all signing functionality into the signed manifeststore.

manifest registration
---------------------

Add a mechanism to register manifest media types and to allow different manifest
types to be Unmarshalled correctly.

client
------

Add ManifestServiceOptions to client functions to allow tags to be passed into Put and
Get for building correct registry URLs.  Change functional arguments to be an interface type
to allow passing data without mutating shared state.

Signed-off-by: Richard Scothern <richard.scothern@gmail.com>

Signed-off-by: Richard Scothern <richard.scothern@docker.com>
This commit is contained in:
Richard Scothern 2015-08-20 21:50:15 -07:00 • committed by Richard Scothern
parent 0fef25389d
commit 8efb9ca329
18 changed files with 1161 additions and 656 deletions
Download patch file Download diff file Expand all files Collapse all files

324
docs/client/repository.go
View file

@ -3,6 +3,7 @@ package client
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -14,7 +15,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/client/transport"
@ -156,26 +156,139 @@ func (r *repository) Manifests(ctx context.Context, options ...distribution.Mani
}, nil
}
func (r *repository) Signatures() distribution.SignatureService {
ms, _ := r.Manifests(r.context)
return &signatures{
manifests: ms,
func (r *repository) Tags(ctx context.Context) distribution.TagService {
return &tags{
client: r.client,
ub: r.ub,
context: r.context,
name: r.Name(),
}
}
type signatures struct {
manifests distribution.ManifestService
// tags implements remote tagging operations.
type tags struct {
client *http.Client
ub *v2.URLBuilder
context context.Context
name string
}
func (s *signatures) Get(dgst digest.Digest) ([][]byte, error) {
m, err := s.manifests.Get(dgst)
// All returns all tags
func (t *tags) All(ctx context.Context) ([]string, error) {
var tags []string
u, err := t.ub.BuildTagsURL(t.name)
if err != nil {
return nil, err
return tags, err
}
return m.Signatures()
resp, err := t.client.Get(u)
if err != nil {
return tags, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return tags, err
}
tags = tagsResponse.Tags
return tags, nil
}
return tags, handleErrorResponse(resp)
}
func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error {
func descriptorFromResponse(response *http.Response) (distribution.Descriptor, error) {
desc := distribution.Descriptor{}
headers := response.Header
ctHeader := headers.Get("Content-Type")
if ctHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Type header")
}
desc.MediaType = ctHeader
digestHeader := headers.Get("Docker-Content-Digest")
if digestHeader == "" {
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return distribution.Descriptor{}, err
}
_, desc, err := distribution.UnmarshalManifest(ctHeader, bytes)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
dgst, err := digest.ParseDigest(digestHeader)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Digest = dgst
lengthHeader := headers.Get("Content-Length")
if lengthHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Length header")
}
length, err := strconv.ParseInt(lengthHeader, 10, 64)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Size = length
return desc, nil
}
// Get issues a HEAD request for a Manifest against its named endpoint in order
// to construct a descriptor for the tag. If the registry doesn't support HEADing
// a manifest, fallback to GET.
func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
u, err := t.ub.BuildManifestURL(t.name, tag)
if err != nil {
return distribution.Descriptor{}, err
}
var attempts int
resp, err := t.client.Head(u)
check:
if err != nil {
return distribution.Descriptor{}, err
}
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 400:
return descriptorFromResponse(resp)
case resp.StatusCode == http.StatusMethodNotAllowed:
resp, err = t.client.Get(u)
attempts++
if attempts > 1 {
return distribution.Descriptor{}, err
}
goto check
default:
return distribution.Descriptor{}, handleErrorResponse(resp)
}
}
func (t *tags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func (t *tags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
panic("not implemented")
}
func (t *tags) Untag(ctx context.Context, tag string) error {
panic("not implemented")
}
@ -186,44 +299,8 @@ type manifests struct {
etags map[string]string
}
func (ms *manifests) Tags() ([]string, error) {
u, err := ms.ub.BuildTagsURL(ms.name)
if err != nil {
return nil, err
}
resp, err := ms.client.Get(u)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return nil, err
}
return tagsResponse.Tags, nil
}
return nil, handleErrorResponse(resp)
}
func (ms *manifests) Exists(dgst digest.Digest) (bool, error) {
// Call by Tag endpoint since the API uses the same
// URL endpoint for tags and digests.
return ms.ExistsByTag(dgst.String())
}
func (ms *manifests) ExistsByTag(tag string) (bool, error) {
u, err := ms.ub.BuildManifestURL(ms.name, tag)
func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
u, err := ms.ub.BuildManifestURL(ms.name, dgst.String())
if err != nil {
return false, err
}
@ -241,46 +318,63 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) {
return false, handleErrorResponse(resp)
}
func (ms *manifests) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
// Call by Tag endpoint since the API uses the same
// URL endpoint for tags and digests.
return ms.GetByTag(dgst.String())
}
// AddEtagToTag allows a client to supply an eTag to GetByTag which will be
// AddEtagToTag allows a client to supply an eTag to Get which will be
// used for a conditional HTTP request. If the eTag matches, a nil manifest
// and nil error will be returned. etag is automatically quoted when added to
// this map.
// and ErrManifestNotModified error will be returned. etag is automatically
// quoted when added to this map.
func AddEtagToTag(tag, etag string) distribution.ManifestServiceOption {
return func(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[tag] = fmt.Sprintf(`"%s"`, etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
return etagOption{tag, etag}
}
func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
type etagOption struct{ tag, etag string }
func (o etagOption) Apply(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[o.tag] = fmt.Sprintf(`"%s"`, o.etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
var tag string
for _, option := range options {
err := option(ms)
if err != nil {
return nil, err
if opt, ok := option.(withTagOption); ok {
tag = opt.tag
} else {
err := option.Apply(ms)
if err != nil {
return nil, err
}
}
}
u, err := ms.ub.BuildManifestURL(ms.name, tag)
var ref string
if tag != "" {
ref = tag
} else {
ref = dgst.String()
}
u, err := ms.ub.BuildManifestURL(ms.name, ref)
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
if _, ok := ms.etags[tag]; ok {
req.Header.Set("If-None-Match", ms.etags[tag])
for _, t := range distribution.ManifestMediaTypes() {
req.Header.Add("Accept", t)
}
if _, ok := ms.etags[ref]; ok {
req.Header.Set("If-None-Match", ms.etags[ref])
}
resp, err := ms.client.Do(req)
if err != nil {
return nil, err
@ -289,44 +383,89 @@ func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServic
if resp.StatusCode == http.StatusNotModified {
return nil, distribution.ErrManifestNotModified
} else if SuccessStatus(resp.StatusCode) {
var sm schema1.SignedManifest
decoder := json.NewDecoder(resp.Body)
mt := resp.Header.Get("Content-Type")
body, err := ioutil.ReadAll(resp.Body)
if err := decoder.Decode(&sm); err != nil {
if err != nil {
return nil, err
}
return &sm, nil
m, _, err := distribution.UnmarshalManifest(mt, body)
if err != nil {
return nil, err
}
return m, nil
}
return nil, handleErrorResponse(resp)
}
func (ms *manifests) Put(m *schema1.SignedManifest) error {
manifestURL, err := ms.ub.BuildManifestURL(ms.name, m.Tag)
if err != nil {
return err
// WithTag allows a tag to be passed into Put which enables the client
// to build a correct URL.
func WithTag(tag string) distribution.ManifestServiceOption {
return withTagOption{tag}
}
type withTagOption struct{ tag string }
func (o withTagOption) Apply(m distribution.ManifestService) error {
if _, ok := m.(*manifests); ok {
return nil
}
return fmt.Errorf("withTagOption is a client-only option")
}
// Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the
// tag name in order to build the correct upload URL. This state is written and read under a lock.
func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
var tag string
for _, option := range options {
if opt, ok := option.(withTagOption); ok {
tag = opt.tag
} else {
err := option.Apply(ms)
if err != nil {
return "", err
}
}
}
// todo(richardscothern): do something with options here when they become applicable
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(m.Raw))
manifestURL, err := ms.ub.BuildManifestURL(ms.name, tag)
if err != nil {
return err
return "", err
}
mediaType, p, err := m.Payload()
if err != nil {
return "", err
}
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(p))
if err != nil {
return "", err
}
putRequest.Header.Set("Content-Type", mediaType)
resp, err := ms.client.Do(putRequest)
if err != nil {
return err
return "", err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
// TODO(dmcgowan): make use of digest header
return nil
dgstHeader := resp.Header.Get("Docker-Content-Digest")
dgst, err := digest.ParseDigest(dgstHeader)
if err != nil {
return "", err
}
return dgst, nil
}
return handleErrorResponse(resp)
return "", handleErrorResponse(resp)
}
func (ms *manifests) Delete(dgst digest.Digest) error {
func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error {
u, err := ms.ub.BuildManifestURL(ms.name, dgst.String())
if err != nil {
return err
@ -348,6 +487,11 @@ func (ms *manifests) Delete(dgst digest.Digest) error {
return handleErrorResponse(resp)
}
// todo(richardscothern): Restore interface and implementation with merge of #1050
/*func (ms *manifests) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
panic("not supported")
}*/
type blobs struct {
name string
ub *v2.URLBuilder

125
docs/client/repository_test.go
View file

@ -42,7 +42,6 @@ func newRandomBlob(size int) (digest.Digest, []byte) {
}
func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.RequestResponseMap) {
*m = append(*m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
@ -499,12 +498,7 @@ func newRandomSchemaV1Manifest(name, tag string, blobCount int) (*schema1.Signed
panic(err)
}
p, err := sm.Payload()
if err != nil {
panic(err)
}
return sm, digest.FromBytes(p), p
return sm, digest.FromBytes(sm.Canonical), sm.Canonical
}
func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil.RequestResponseMap, dgst string) {
@ -525,6 +519,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil
Headers: http.Header(map[string][]string{
"Content-Length": {"0"},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
}
} else {
@ -534,6 +529,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
}
@ -553,6 +549,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
},
})
@ -566,6 +563,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
},
})
@ -598,12 +596,17 @@ func checkEqualManifest(m1, m2 *schema1.SignedManifest) error {
return nil
}
func TestManifestFetch(t *testing.T) {
func TestV1ManifestFetch(t *testing.T) {
ctx := context.Background()
repo := "test.example.com/repo"
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "latest", 6)
var m testutil.RequestResponseMap
addTestManifest(repo, dgst.String(), m1.Raw, &m)
_, pl, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
addTestManifest(repo, dgst.String(), pl, &m)
addTestManifest(repo, "latest", pl, &m)
e, c := testServer(m)
defer c()
@ -617,7 +620,7 @@ func TestManifestFetch(t *testing.T) {
t.Fatal(err)
}
ok, err := ms.Exists(dgst)
ok, err := ms.Exists(ctx, dgst)
if err != nil {
t.Fatal(err)
}
@ -625,11 +628,29 @@ func TestManifestFetch(t *testing.T) {
t.Fatal("Manifest does not exist")
}
manifest, err := ms.Get(dgst)
manifest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatal(err)
}
if err := checkEqualManifest(manifest, m1); err != nil {
v1manifest, ok := manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err := checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
manifest, err = ms.Get(ctx, dgst, WithTag("latest"))
if err != nil {
t.Fatal(err)
}
v1manifest, ok = manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err = checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
}
@ -643,17 +664,22 @@ func TestManifestFetchWithEtag(t *testing.T) {
e, c := testServer(m)
defer c()
r, err := NewRepository(context.Background(), repo, e, nil)
ctx := context.Background()
r, err := NewRepository(ctx, repo, e, nil)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
_, err = ms.GetByTag("latest", AddEtagToTag("latest", d1.String()))
clientManifestService, ok := ms.(*manifests)
if !ok {
panic("wrong type for client manifest service")
}
_, err = clientManifestService.Get(ctx, d1, WithTag("latest"), AddEtagToTag("latest", d1.String()))
if err != distribution.ErrManifestNotModified {
t.Fatal(err)
}
@ -690,10 +716,10 @@ func TestManifestDelete(t *testing.T) {
t.Fatal(err)
}
if err := ms.Delete(dgst1); err != nil {
if err := ms.Delete(ctx, dgst1); err != nil {
t.Fatal(err)
}
if err := ms.Delete(dgst2); err == nil {
if err := ms.Delete(ctx, dgst2); err == nil {
t.Fatal("Expected error deleting unknown manifest")
}
// TODO(dmcgowan): Check for specific unknown error
@ -702,12 +728,17 @@ func TestManifestDelete(t *testing.T) {
func TestManifestPut(t *testing.T) {
repo := "test.example.com/repo/delete"
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "other", 6)
_, payload, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "PUT",
Route: "/v2/" + repo + "/manifests/other",
Body: m1.Raw,
Body: payload,
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
@ -731,7 +762,7 @@ func TestManifestPut(t *testing.T) {
t.Fatal(err)
}
if err := ms.Put(m1); err != nil {
if _, err := ms.Put(ctx, m1, WithTag(m1.Tag)); err != nil {
t.Fatal(err)
}
@ -751,21 +782,22 @@ func TestManifestTags(t *testing.T) {
}
`))
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/" + repo + "/tags/list",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: tagsList,
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(tagsList))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
}),
},
})
for i := 0; i < 3; i++ {
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/" + repo + "/tags/list",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: tagsList,
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(tagsList))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
}),
},
})
}
e, c := testServer(m)
defer c()
@ -773,22 +805,29 @@ func TestManifestTags(t *testing.T) {
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
tagService := r.Tags(ctx)
tags, err := tagService.All(ctx)
if err != nil {
t.Fatal(err)
}
tags, err := ms.Tags()
if err != nil {
t.Fatal(err)
}
if len(tags) != 3 {
t.Fatalf("Wrong number of tags returned: %d, expected 3", len(tags))
}
// TODO(dmcgowan): Check array
expected := map[string]struct{}{
"tag1": {},
"tag2": {},
"funtag": {},
}
for _, t := range tags {
delete(expected, t)
}
if len(expected) != 0 {
t.Fatalf("unexpected tags returned: %v", expected)
}
// TODO(dmcgowan): Check for error cases
}
@ -821,7 +860,7 @@ func TestManifestUnauthorized(t *testing.T) {
t.Fatal(err)
}
_, err = ms.Get(dgst)
_, err = ms.Get(ctx, dgst)
if err == nil {
t.Fatal("Expected error fetching manifest")
}