vbatts/registry
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Implementation of the Manifest Service API refactor.

Browse source 
Add a generic Manifest interface to represent manifests in the registry and
remove references to schema specific manifests.

Add a ManifestBuilder to construct Manifest objects. Concrete manifest builders
will exist for each manifest type and implementations will contain manifest
specific data used to build a manifest.

Remove Signatures() from Repository interface.

Signatures are relevant only to schema1 manifests.  Move access to the signature
store inside the schema1 manifestStore.  Add some API tests to verify
signature roundtripping.

schema1
-------

Change the way data is stored in schema1.Manifest to enable Payload() to be used
to return complete Manifest JSON from the HTTP handler without knowledge of the
schema1 protocol.

tags
----

Move tag functionality to a seperate TagService and update ManifestService
to use the new interfaces.  Implement a driver based tagService to be backward
compatible with the current tag service.

Add a proxyTagService to enable the registry to get a digest for remote manifests
from a tag.

manifest store
--------------

Remove revision store and move all signing functionality into the signed manifeststore.

manifest registration
---------------------

Add a mechanism to register manifest media types and to allow different manifest
types to be Unmarshalled correctly.

client
------

Add ManifestServiceOptions to client functions to allow tags to be passed into Put and
Get for building correct registry URLs.  Change functional arguments to be an interface type
to allow passing data without mutating shared state.

Signed-off-by: Richard Scothern <richard.scothern@gmail.com>

Signed-off-by: Richard Scothern <richard.scothern@docker.com>
This commit is contained in:
Richard Scothern 2015-08-20 21:50:15 -07:00 committed by Richard Scothern
parent 0fef25389d
commit 8efb9ca329
18 changed files with 1161 additions and 656 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/api/v2/descriptors.go
View file

@ -495,7 +495,7 @@ var routeDescriptors = []RouteDescriptor{
Methods: []MethodDescriptor{
{
Method: "GET",
Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest.",
Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data.",
Requests: []RequestDescriptor{
{
Headers: []ParameterDescriptor{

324
docs/client/repository.go
View file

@ -3,6 +3,7 @@ package client
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@ -14,7 +15,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/api/v2"
"github.com/docker/distribution/registry/client/transport"
@ -156,26 +156,139 @@ func (r *repository) Manifests(ctx context.Context, options ...distribution.Mani
}, nil
}
func (r *repository) Signatures() distribution.SignatureService {
ms, _ := r.Manifests(r.context)
return &signatures{
manifests: ms,
func (r *repository) Tags(ctx context.Context) distribution.TagService {
return &tags{
client: r.client,
ub: r.ub,
context: r.context,
name: r.Name(),
}
}
type signatures struct {
manifests distribution.ManifestService
// tags implements remote tagging operations.
type tags struct {
client *http.Client
ub *v2.URLBuilder
context context.Context
name string
}
func (s *signatures) Get(dgst digest.Digest) ([][]byte, error) {
m, err := s.manifests.Get(dgst)
// All returns all tags
func (t *tags) All(ctx context.Context) ([]string, error) {
var tags []string
u, err := t.ub.BuildTagsURL(t.name)
if err != nil {
return nil, err
return tags, err
}
return m.Signatures()
resp, err := t.client.Get(u)
if err != nil {
return tags, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return tags, err
}
tags = tagsResponse.Tags
return tags, nil
}
return tags, handleErrorResponse(resp)
}
func (s *signatures) Put(dgst digest.Digest, signatures ...[]byte) error {
func descriptorFromResponse(response *http.Response) (distribution.Descriptor, error) {
desc := distribution.Descriptor{}
headers := response.Header
ctHeader := headers.Get("Content-Type")
if ctHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Type header")
}
desc.MediaType = ctHeader
digestHeader := headers.Get("Docker-Content-Digest")
if digestHeader == "" {
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return distribution.Descriptor{}, err
}
_, desc, err := distribution.UnmarshalManifest(ctHeader, bytes)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
dgst, err := digest.ParseDigest(digestHeader)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Digest = dgst
lengthHeader := headers.Get("Content-Length")
if lengthHeader == "" {
return distribution.Descriptor{}, errors.New("missing or empty Content-Length header")
}
length, err := strconv.ParseInt(lengthHeader, 10, 64)
if err != nil {
return distribution.Descriptor{}, err
}
desc.Size = length
return desc, nil
}
// Get issues a HEAD request for a Manifest against its named endpoint in order
// to construct a descriptor for the tag. If the registry doesn't support HEADing
// a manifest, fallback to GET.
func (t *tags) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
u, err := t.ub.BuildManifestURL(t.name, tag)
if err != nil {
return distribution.Descriptor{}, err
}
var attempts int
resp, err := t.client.Head(u)
check:
if err != nil {
return distribution.Descriptor{}, err
}
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 400:
return descriptorFromResponse(resp)
case resp.StatusCode == http.StatusMethodNotAllowed:
resp, err = t.client.Get(u)
attempts++
if attempts > 1 {
return distribution.Descriptor{}, err
}
goto check
default:
return distribution.Descriptor{}, handleErrorResponse(resp)
}
}
func (t *tags) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func (t *tags) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
panic("not implemented")
}
func (t *tags) Untag(ctx context.Context, tag string) error {
panic("not implemented")
}
@ -186,44 +299,8 @@ type manifests struct {
etags map[string]string
}
func (ms *manifests) Tags() ([]string, error) {
u, err := ms.ub.BuildTagsURL(ms.name)
if err != nil {
return nil, err
}
resp, err := ms.client.Get(u)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
tagsResponse := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResponse); err != nil {
return nil, err
}
return tagsResponse.Tags, nil
}
return nil, handleErrorResponse(resp)
}
func (ms *manifests) Exists(dgst digest.Digest) (bool, error) {
// Call by Tag endpoint since the API uses the same
// URL endpoint for tags and digests.
return ms.ExistsByTag(dgst.String())
}
func (ms *manifests) ExistsByTag(tag string) (bool, error) {
u, err := ms.ub.BuildManifestURL(ms.name, tag)
func (ms *manifests) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
u, err := ms.ub.BuildManifestURL(ms.name, dgst.String())
if err != nil {
return false, err
}
@ -241,46 +318,63 @@ func (ms *manifests) ExistsByTag(tag string) (bool, error) {
return false, handleErrorResponse(resp)
}
func (ms *manifests) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
// Call by Tag endpoint since the API uses the same
// URL endpoint for tags and digests.
return ms.GetByTag(dgst.String())
}
// AddEtagToTag allows a client to supply an eTag to GetByTag which will be
// AddEtagToTag allows a client to supply an eTag to Get which will be
// used for a conditional HTTP request. If the eTag matches, a nil manifest
// and nil error will be returned. etag is automatically quoted when added to
// this map.
// and ErrManifestNotModified error will be returned. etag is automatically
// quoted when added to this map.
func AddEtagToTag(tag, etag string) distribution.ManifestServiceOption {
return func(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[tag] = fmt.Sprintf(`"%s"`, etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
return etagOption{tag, etag}
}
func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
type etagOption struct{ tag, etag string }
func (o etagOption) Apply(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifests); ok {
ms.etags[o.tag] = fmt.Sprintf(`"%s"`, o.etag)
return nil
}
return fmt.Errorf("etag options is a client-only option")
}
func (ms *manifests) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
var tag string
for _, option := range options {
err := option(ms)
if err != nil {
return nil, err
if opt, ok := option.(withTagOption); ok {
tag = opt.tag
} else {
err := option.Apply(ms)
if err != nil {
return nil, err
}
}
}
u, err := ms.ub.BuildManifestURL(ms.name, tag)
var ref string
if tag != "" {
ref = tag
} else {
ref = dgst.String()
}
u, err := ms.ub.BuildManifestURL(ms.name, ref)
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
if _, ok := ms.etags[tag]; ok {
req.Header.Set("If-None-Match", ms.etags[tag])
for _, t := range distribution.ManifestMediaTypes() {
req.Header.Add("Accept", t)
}
if _, ok := ms.etags[ref]; ok {
req.Header.Set("If-None-Match", ms.etags[ref])
}
resp, err := ms.client.Do(req)
if err != nil {
return nil, err
@ -289,44 +383,89 @@ func (ms *manifests) GetByTag(tag string, options ...distribution.ManifestServic
if resp.StatusCode == http.StatusNotModified {
return nil, distribution.ErrManifestNotModified
} else if SuccessStatus(resp.StatusCode) {
var sm schema1.SignedManifest
decoder := json.NewDecoder(resp.Body)
mt := resp.Header.Get("Content-Type")
body, err := ioutil.ReadAll(resp.Body)
if err := decoder.Decode(&sm); err != nil {
if err != nil {
return nil, err
}
return &sm, nil
m, _, err := distribution.UnmarshalManifest(mt, body)
if err != nil {
return nil, err
}
return m, nil
}
return nil, handleErrorResponse(resp)
}
func (ms *manifests) Put(m *schema1.SignedManifest) error {
manifestURL, err := ms.ub.BuildManifestURL(ms.name, m.Tag)
if err != nil {
return err
// WithTag allows a tag to be passed into Put which enables the client
// to build a correct URL.
func WithTag(tag string) distribution.ManifestServiceOption {
return withTagOption{tag}
}
type withTagOption struct{ tag string }
func (o withTagOption) Apply(m distribution.ManifestService) error {
if _, ok := m.(*manifests); ok {
return nil
}
return fmt.Errorf("withTagOption is a client-only option")
}
// Put puts a manifest. A tag can be specified using an options parameter which uses some shared state to hold the
// tag name in order to build the correct upload URL. This state is written and read under a lock.
func (ms *manifests) Put(ctx context.Context, m distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
var tag string
for _, option := range options {
if opt, ok := option.(withTagOption); ok {
tag = opt.tag
} else {
err := option.Apply(ms)
if err != nil {
return "", err
}
}
}
// todo(richardscothern): do something with options here when they become applicable
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(m.Raw))
manifestURL, err := ms.ub.BuildManifestURL(ms.name, tag)
if err != nil {
return err
return "", err
}
mediaType, p, err := m.Payload()
if err != nil {
return "", err
}
putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(p))
if err != nil {
return "", err
}
putRequest.Header.Set("Content-Type", mediaType)
resp, err := ms.client.Do(putRequest)
if err != nil {
return err
return "", err
}
defer resp.Body.Close()
if SuccessStatus(resp.StatusCode) {
// TODO(dmcgowan): make use of digest header
return nil
dgstHeader := resp.Header.Get("Docker-Content-Digest")
dgst, err := digest.ParseDigest(dgstHeader)
if err != nil {
return "", err
}
return dgst, nil
}
return handleErrorResponse(resp)
return "", handleErrorResponse(resp)
}
func (ms *manifests) Delete(dgst digest.Digest) error {
func (ms *manifests) Delete(ctx context.Context, dgst digest.Digest) error {
u, err := ms.ub.BuildManifestURL(ms.name, dgst.String())
if err != nil {
return err
@ -348,6 +487,11 @@ func (ms *manifests) Delete(dgst digest.Digest) error {
return handleErrorResponse(resp)
}
// todo(richardscothern): Restore interface and implementation with merge of #1050
/*func (ms *manifests) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
panic("not supported")
}*/
type blobs struct {
name string
ub *v2.URLBuilder

125
docs/client/repository_test.go
View file

@ -42,7 +42,6 @@ func newRandomBlob(size int) (digest.Digest, []byte) {
}
func addTestFetch(repo string, dgst digest.Digest, content []byte, m *testutil.RequestResponseMap) {
*m = append(*m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
@ -499,12 +498,7 @@ func newRandomSchemaV1Manifest(name, tag string, blobCount int) (*schema1.Signed
panic(err)
}
p, err := sm.Payload()
if err != nil {
panic(err)
}
return sm, digest.FromBytes(p), p
return sm, digest.FromBytes(sm.Canonical), sm.Canonical
}
func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil.RequestResponseMap, dgst string) {
@ -525,6 +519,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil
Headers: http.Header(map[string][]string{
"Content-Length": {"0"},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
}
} else {
@ -534,6 +529,7 @@ func addTestManifestWithEtag(repo, reference string, content []byte, m *testutil
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
}
@ -553,6 +549,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
},
})
@ -566,6 +563,7 @@ func addTestManifest(repo, reference string, content []byte, m *testutil.Request
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(content))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
"Content-Type": {schema1.MediaTypeManifest},
}),
},
})
@ -598,12 +596,17 @@ func checkEqualManifest(m1, m2 *schema1.SignedManifest) error {
return nil
}
func TestManifestFetch(t *testing.T) {
func TestV1ManifestFetch(t *testing.T) {
ctx := context.Background()
repo := "test.example.com/repo"
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "latest", 6)
var m testutil.RequestResponseMap
addTestManifest(repo, dgst.String(), m1.Raw, &m)
_, pl, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
addTestManifest(repo, dgst.String(), pl, &m)
addTestManifest(repo, "latest", pl, &m)
e, c := testServer(m)
defer c()
@ -617,7 +620,7 @@ func TestManifestFetch(t *testing.T) {
t.Fatal(err)
}
ok, err := ms.Exists(dgst)
ok, err := ms.Exists(ctx, dgst)
if err != nil {
t.Fatal(err)
}
@ -625,11 +628,29 @@ func TestManifestFetch(t *testing.T) {
t.Fatal("Manifest does not exist")
}
manifest, err := ms.Get(dgst)
manifest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatal(err)
}
if err := checkEqualManifest(manifest, m1); err != nil {
v1manifest, ok := manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err := checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
manifest, err = ms.Get(ctx, dgst, WithTag("latest"))
if err != nil {
t.Fatal(err)
}
v1manifest, ok = manifest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("Unexpected manifest type from Get: %T", manifest)
}
if err = checkEqualManifest(v1manifest, m1); err != nil {
t.Fatal(err)
}
}
@ -643,17 +664,22 @@ func TestManifestFetchWithEtag(t *testing.T) {
e, c := testServer(m)
defer c()
r, err := NewRepository(context.Background(), repo, e, nil)
ctx := context.Background()
r, err := NewRepository(ctx, repo, e, nil)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
_, err = ms.GetByTag("latest", AddEtagToTag("latest", d1.String()))
clientManifestService, ok := ms.(*manifests)
if !ok {
panic("wrong type for client manifest service")
}
_, err = clientManifestService.Get(ctx, d1, WithTag("latest"), AddEtagToTag("latest", d1.String()))
if err != distribution.ErrManifestNotModified {
t.Fatal(err)
}
@ -690,10 +716,10 @@ func TestManifestDelete(t *testing.T) {
t.Fatal(err)
}
if err := ms.Delete(dgst1); err != nil {
if err := ms.Delete(ctx, dgst1); err != nil {
t.Fatal(err)
}
if err := ms.Delete(dgst2); err == nil {
if err := ms.Delete(ctx, dgst2); err == nil {
t.Fatal("Expected error deleting unknown manifest")
}
// TODO(dmcgowan): Check for specific unknown error
@ -702,12 +728,17 @@ func TestManifestDelete(t *testing.T) {
func TestManifestPut(t *testing.T) {
repo := "test.example.com/repo/delete"
m1, dgst, _ := newRandomSchemaV1Manifest(repo, "other", 6)
_, payload, err := m1.Payload()
if err != nil {
t.Fatal(err)
}
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "PUT",
Route: "/v2/" + repo + "/manifests/other",
Body: m1.Raw,
Body: payload,
},
Response: testutil.Response{
StatusCode: http.StatusAccepted,
@ -731,7 +762,7 @@ func TestManifestPut(t *testing.T) {
t.Fatal(err)
}
if err := ms.Put(m1); err != nil {
if _, err := ms.Put(ctx, m1, WithTag(m1.Tag)); err != nil {
t.Fatal(err)
}
@ -751,21 +782,22 @@ func TestManifestTags(t *testing.T) {
}
`))
var m testutil.RequestResponseMap
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/" + repo + "/tags/list",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: tagsList,
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(tagsList))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
}),
},
})
for i := 0; i < 3; i++ {
m = append(m, testutil.RequestResponseMapping{
Request: testutil.Request{
Method: "GET",
Route: "/v2/" + repo + "/tags/list",
},
Response: testutil.Response{
StatusCode: http.StatusOK,
Body: tagsList,
Headers: http.Header(map[string][]string{
"Content-Length": {fmt.Sprint(len(tagsList))},
"Last-Modified": {time.Now().Add(-1 * time.Second).Format(time.ANSIC)},
}),
},
})
}
e, c := testServer(m)
defer c()
@ -773,22 +805,29 @@ func TestManifestTags(t *testing.T) {
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
ms, err := r.Manifests(ctx)
tagService := r.Tags(ctx)
tags, err := tagService.All(ctx)
if err != nil {
t.Fatal(err)
}
tags, err := ms.Tags()
if err != nil {
t.Fatal(err)
}
if len(tags) != 3 {
t.Fatalf("Wrong number of tags returned: %d, expected 3", len(tags))
}
// TODO(dmcgowan): Check array
expected := map[string]struct{}{
"tag1": {},
"tag2": {},
"funtag": {},
}
for _, t := range tags {
delete(expected, t)
}
if len(expected) != 0 {
t.Fatalf("unexpected tags returned: %v", expected)
}
// TODO(dmcgowan): Check for error cases
}
@ -821,7 +860,7 @@ func TestManifestUnauthorized(t *testing.T) {
t.Fatal(err)
}
_, err = ms.Get(dgst)
_, err = ms.Get(ctx, dgst)
if err == nil {
t.Fatal("Expected error fetching manifest")
}

149
docs/handlers/api_test.go
View file

@ -871,19 +871,15 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst := digest.FromBytes(payload)
dgst := digest.FromBytes(signedManifest.Canonical)
args.signedManifest = signedManifest
args.dgst = dgst
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
resp = putManifest(t, "putting signed manifest no error", manifestURL, signedManifest)
checkResponse(t, "putting signed manifest no error", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
@ -914,11 +910,12 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
var fetchedManifest schema1.SignedManifest
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) {
if !bytes.Equal(fetchedManifest.Canonical, signedManifest.Canonical) {
t.Fatalf("manifests do not match")
}
@ -940,10 +937,55 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifestByDigest.Raw, signedManifest.Raw) {
if !bytes.Equal(fetchedManifestByDigest.Canonical, signedManifest.Canonical) {
t.Fatalf("manifests do not match")
}
// check signature was roundtripped
signatures, err := fetchedManifestByDigest.Signatures()
if err != nil {
t.Fatal(err)
}
if len(signatures) != 1 {
t.Fatalf("expected 1 signature from manifest, got: %d", len(signatures))
}
// Re-sign, push and pull the same digest
sm2, err := schema1.Sign(&fetchedManifestByDigest.Manifest, env.pk)
if err != nil {
t.Fatal(err)
}
resp = putManifest(t, "re-putting signed manifest", manifestDigestURL, sm2)
checkResponse(t, "re-putting signed manifest", resp, http.StatusCreated)
resp, err = http.Get(manifestDigestURL)
checkErr(t, err, "re-fetching manifest by digest")
defer resp.Body.Close()
checkResponse(t, "re-fetching uploaded manifest", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
"ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
})
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifestByDigest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
// check two signatures were roundtripped
signatures, err = fetchedManifestByDigest.Signatures()
if err != nil {
t.Fatal(err)
}
if len(signatures) != 2 {
t.Fatalf("expected 2 signature from manifest, got: %d", len(signatures))
}
// Get by name with etag, gives 304
etag := resp.Header.Get("Etag")
req, err := http.NewRequest("GET", manifestURL, nil)
@ -956,7 +998,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified)
checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified)
// Get by digest with etag, gives 304
req, err = http.NewRequest("GET", manifestDigestURL, nil)
@ -969,7 +1011,7 @@ func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, m
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified)
checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified)
// Ensure that the tag is listed.
resp, err = http.Get(tagsURL)
@ -1143,8 +1185,13 @@ func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *te
func putManifest(t *testing.T, msg, url string, v interface{}) *http.Response {
var body []byte
if sm, ok := v.(*schema1.SignedManifest); ok {
body = sm.Raw
_, pl, err := sm.Payload()
if err != nil {
t.Fatalf("error getting payload: %v", err)
}
body = pl
} else {
var err error
body, err = json.MarshalIndent(v, "", " ")
@ -1435,7 +1482,7 @@ func checkErr(t *testing.T, err error, msg string) {
}
}
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) {
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) digest.Digest {
unsignedManifest := &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
@ -1459,7 +1506,6 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
@ -1477,20 +1523,22 @@ func createRepository(env *testEnv, t *testing.T, imageName string, tag string)
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst := digest.FromBytes(signedManifest.Canonical)
dgst := digest.FromBytes(payload)
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
// Create this repository by tag to ensure the tag mapping is made in the registry
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, tag)
checkErr(t, err, "building manifest url")
location, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building location URL")
resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Location": []string{location},
"Docker-Content-Digest": []string{dgst.String()},
})
return dgst
}
// Test mutation operations on a registry configured as a cache. Ensure that they return
@ -1577,3 +1625,64 @@ func TestCheckContextNotifier(t *testing.T) {
t.Fatalf("wrong status code - expected 200, got %d", resp.StatusCode)
}
}
func TestProxyManifestGetByTag(t *testing.T) {
truthConfig := configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
}
truthConfig.HTTP.Headers = headerConfig
imageName := "foo/bar"
tag := "latest"
truthEnv := newTestEnvWithConfig(t, &truthConfig)
// create a repository in the truth registry
dgst := createRepository(truthEnv, t, imageName, tag)
proxyConfig := configuration.Configuration{
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
},
Proxy: configuration.Proxy{
RemoteURL: truthEnv.server.URL,
},
}
proxyConfig.HTTP.Headers = headerConfig
proxyEnv := newTestEnvWithConfig(t, &proxyConfig)
manifestDigestURL, err := proxyEnv.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp, err := http.Get(manifestDigestURL)
checkErr(t, err, "fetching manifest from proxy by digest")
defer resp.Body.Close()
manifestTagURL, err := proxyEnv.builder.BuildManifestURL(imageName, tag)
checkErr(t, err, "building manifest url")
resp, err = http.Get(manifestTagURL)
checkErr(t, err, "fetching manifest from proxy by tag")
defer resp.Body.Close()
checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
})
// Create another manifest in the remote with the same image/tag pair
newDigest := createRepository(truthEnv, t, imageName, tag)
if dgst == newDigest {
t.Fatalf("non-random test data")
}
// fetch it with the same proxy URL as before. Ensure the updated content is at the same tag
resp, err = http.Get(manifestTagURL)
checkErr(t, err, "fetching manifest from proxy by tag")
defer resp.Body.Close()
checkResponse(t, "fetching manifest from proxy by tag", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{newDigest.String()},
})
}

114
docs/handlers/images.go
View file

@ -2,19 +2,15 @@ package handlers
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/docker/distribution"
ctxu "github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/registry/api/errcode"
"github.com/docker/distribution/registry/api/v2"
"github.com/gorilla/handlers"
"golang.org/x/net/context"
)
// imageManifestDispatcher takes the request context and builds the
@ -33,7 +29,8 @@ func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler {
}
mhandler := handlers.MethodHandler{
"GET": http.HandlerFunc(imageManifestHandler.GetImageManifest),
"GET": http.HandlerFunc(imageManifestHandler.GetImageManifest),
"HEAD": http.HandlerFunc(imageManifestHandler.GetImageManifest),
}
if !ctx.readOnly {
@ -54,6 +51,8 @@ type imageManifestHandler struct {
}
// GetImageManifest fetches the image manifest from the storage backend, if it exists.
// todo(richardscothern): this assumes v2 schema 1 manifests for now but in the future
// get the version from the Accept HTTP header
func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("GetImageManifest")
manifests, err := imh.Repository.Manifests(imh)
@ -62,42 +61,38 @@ func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http
return
}
var sm *schema1.SignedManifest
var manifest distribution.Manifest
if imh.Tag != "" {
sm, err = manifests.GetByTag(imh.Tag)
} else {
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
tags := imh.Repository.Tags(imh)
desc, err := tags.Get(imh, imh.Tag)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
sm, err = manifests.Get(imh.Digest)
imh.Digest = desc.Digest
}
if etagMatch(r, imh.Digest.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
manifest, err = manifests.Get(imh, imh.Digest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestUnknown.WithDetail(err))
return
}
// Get the digest, if we don't already have it.
if imh.Digest == "" {
dgst, err := digestManifest(imh, sm)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
return
}
if etagMatch(r, dgst.String()) {
w.WriteHeader(http.StatusNotModified)
return
}
imh.Digest = dgst
ct, p, err := manifest.Payload()
if err != nil {
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Content-Length", fmt.Sprint(len(sm.Raw)))
w.Header().Set("Content-Type", ct)
w.Header().Set("Content-Length", fmt.Sprint(len(p)))
w.Header().Set("Docker-Content-Digest", imh.Digest.String())
w.Header().Set("Etag", fmt.Sprintf(`"%s"`, imh.Digest))
w.Write(sm.Raw)
w.Write(p)
}
func etagMatch(r *http.Request, etag string) bool {
@ -109,7 +104,7 @@ func etagMatch(r *http.Request, etag string) bool {
return false
}
// PutImageManifest validates and stores and image in the registry.
// PutImageManifest validates and stores an image in the registry.
func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) {
ctxu.GetLogger(imh).Debug("PutImageManifest")
manifests, err := imh.Repository.Manifests(imh)
@ -124,39 +119,28 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http
return
}
var manifest schema1.SignedManifest
if err := json.Unmarshal(jsonBuf.Bytes(), &manifest); err != nil {
mediaType := r.Header.Get("Content-Type")
manifest, desc, err := distribution.UnmarshalManifest(mediaType, jsonBuf.Bytes())
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeManifestInvalid.WithDetail(err))
return
}
dgst, err := digestManifest(imh, &manifest)
if err != nil {
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid.WithDetail(err))
return
}
// Validate manifest tag or digest matches payload
if imh.Tag != "" {
if manifest.Tag != imh.Tag {
ctxu.GetLogger(imh).Errorf("invalid tag on manifest payload: %q != %q", manifest.Tag, imh.Tag)
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid)
return
}
imh.Digest = dgst
} else if imh.Digest != "" {
if dgst != imh.Digest {
ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", dgst, imh.Digest)
if imh.Digest != "" {
if desc.Digest != imh.Digest {
ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", desc.Digest, imh.Digest)
imh.Errors = append(imh.Errors, v2.ErrorCodeDigestInvalid)
return
}
} else if imh.Tag != "" {
imh.Digest = desc.Digest
} else {
imh.Errors = append(imh.Errors, v2.ErrorCodeTagInvalid.WithDetail("no tag or digest specified"))
return
}
if err := manifests.Put(&manifest); err != nil {
_, err = manifests.Put(imh, manifest)
if err != nil {
// TODO(stevvooe): These error handling switches really need to be
// handled by an app global mapper.
if err == distribution.ErrUnsupported {
@ -188,6 +172,17 @@ func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http
return
}
// Tag this manifest
if imh.Tag != "" {
tags := imh.Repository.Tags(imh)
err = tags.Tag(imh, imh.Tag, desc)
if err != nil {
imh.Errors = append(imh.Errors, errcode.ErrorCodeUnknown.WithDetail(err))
return
}
}
// Construct a canonical url for the uploaded manifest.
location, err := imh.urlBuilder.BuildManifestURL(imh.Repository.Name(), imh.Digest.String())
if err != nil {
@ -212,7 +207,7 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h
return
}
err = manifests.Delete(imh.Digest)
err = manifests.Delete(imh, imh.Digest)
if err != nil {
switch err {
case digest.ErrDigestUnsupported:
@ -233,22 +228,3 @@ func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *h
w.WriteHeader(http.StatusAccepted)
}
// digestManifest takes a digest of the given manifest. This belongs somewhere
// better but we'll wait for a refactoring cycle to find that real somewhere.
func digestManifest(ctx context.Context, sm *schema1.SignedManifest) (digest.Digest, error) {
p, err := sm.Payload()
if err != nil {
if !strings.Contains(err.Error(), "missing signature key") {
ctxu.GetLogger(ctx).Errorf("error getting manifest payload: %v", err)
return "", err
}
// NOTE(stevvooe): There are no signatures but we still have a
// payload. The request will fail later but this is not the
// responsibility of this part of the code.
p = sm.Raw
}
return digest.FromBytes(p), nil
}

8
docs/handlers/tags.go
View file

@ -34,13 +34,9 @@ type tagsAPIResponse struct {
// GetTags returns a json list of tags for a specific image name.
func (th *tagsHandler) GetTags(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
manifests, err := th.Repository.Manifests(th)
if err != nil {
th.Errors = append(th.Errors, err)
return
}
tags, err := manifests.Tags()
tagService := th.Repository.Tags(th)
tags, err := tagService.All(th)
if err != nil {
switch err := err.(type) {
case distribution.ErrRepositoryUnknown:

137
docs/proxy/proxymanifeststore.go
View file

@ -6,8 +6,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/registry/client"
"github.com/docker/distribution/registry/proxy/scheduler"
)
@ -24,8 +22,8 @@ type proxyManifestStore struct {
var _ distribution.ManifestService = &proxyManifestStore{}
func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) {
exists, err := pms.localManifests.Exists(dgst)
func (pms proxyManifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
exists, err := pms.localManifests.Exists(ctx, dgst)
if err != nil {
return false, err
}
@ -33,117 +31,56 @@ func (pms proxyManifestStore) Exists(dgst digest.Digest) (bool, error) {
return true, nil
}
return pms.remoteManifests.Exists(dgst)
return pms.remoteManifests.Exists(ctx, dgst)
}
func (pms proxyManifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
sm, err := pms.localManifests.Get(dgst)
if err == nil {
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
func (pms proxyManifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
// At this point `dgst` was either specified explicitly, or returned by the
// tagstore with the most recent association.
var fromRemote bool
manifest, err := pms.localManifests.Get(ctx, dgst, options...)
if err != nil {
manifest, err = pms.remoteManifests.Get(ctx, dgst, options...)
if err != nil {
return nil, err
}
fromRemote = true
}
sm, err = pms.remoteManifests.Get(dgst)
_, payload, err := manifest.Payload()
if err != nil {
return nil, err
}
proxyMetrics.ManifestPull(uint64(len(sm.Raw)))
err = pms.localManifests.Put(sm)
if err != nil {
return nil, err
proxyMetrics.ManifestPush(uint64(len(payload)))
if fromRemote {
proxyMetrics.ManifestPull(uint64(len(payload)))
_, err = pms.localManifests.Put(ctx, manifest)
if err != nil {
return nil, err
}
// Schedule the repo for removal
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
// Ensure the manifest blob is cleaned up
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
}
// Schedule the repo for removal
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
// Ensure the manifest blob is cleaned up
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
return manifest, err
}
func (pms proxyManifestStore) Tags() ([]string, error) {
return pms.localManifests.Tags()
func (pms proxyManifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
var d digest.Digest
return d, distribution.ErrUnsupported
}
func (pms proxyManifestStore) ExistsByTag(tag string) (bool, error) {
exists, err := pms.localManifests.ExistsByTag(tag)
if err != nil {
return false, err
}
if exists {
return true, nil
}
return pms.remoteManifests.ExistsByTag(tag)
}
func (pms proxyManifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
var localDigest digest.Digest
localManifest, err := pms.localManifests.GetByTag(tag, options...)
switch err.(type) {
case distribution.ErrManifestUnknown, distribution.ErrManifestUnknownRevision:
goto fromremote
case nil:
break
default:
return nil, err
}
localDigest, err = manifestDigest(localManifest)
if err != nil {
return nil, err
}
fromremote:
var sm *schema1.SignedManifest
sm, err = pms.remoteManifests.GetByTag(tag, client.AddEtagToTag(tag, localDigest.String()))
if err != nil && err != distribution.ErrManifestNotModified {
return nil, err
}
if err == distribution.ErrManifestNotModified {
context.GetLogger(pms.ctx).Debugf("Local manifest for %q is latest, dgst=%s", tag, localDigest.String())
return localManifest, nil
}
context.GetLogger(pms.ctx).Debugf("Updated manifest for %q, dgst=%s", tag, localDigest.String())
err = pms.localManifests.Put(sm)
if err != nil {
return nil, err
}
dgst, err := manifestDigest(sm)
if err != nil {
return nil, err
}
pms.scheduler.AddBlob(dgst.String(), repositoryTTL)
pms.scheduler.AddManifest(pms.repositoryName, repositoryTTL)
proxyMetrics.ManifestPull(uint64(len(sm.Raw)))
proxyMetrics.ManifestPush(uint64(len(sm.Raw)))
return sm, err
}
func manifestDigest(sm *schema1.SignedManifest) (digest.Digest, error) {
payload, err := sm.Payload()
if err != nil {
return "", err
}
return digest.FromBytes(payload), nil
}
func (pms proxyManifestStore) Put(manifest *schema1.SignedManifest) error {
func (pms proxyManifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
return distribution.ErrUnsupported
}
func (pms proxyManifestStore) Delete(dgst digest.Digest) error {
return distribution.ErrUnsupported
/*func (pms proxyManifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
return 0, distribution.ErrUnsupported
}
*/

64
docs/proxy/proxymanifeststore_test.go
View file

@ -37,40 +37,31 @@ func (te manifestStoreTestEnv) RemoteStats() *map[string]int {
return &rs
}
func (sm statsManifest) Delete(dgst digest.Digest) error {
func (sm statsManifest) Delete(ctx context.Context, dgst digest.Digest) error {
sm.stats["delete"]++
return sm.manifests.Delete(dgst)
return sm.manifests.Delete(ctx, dgst)
}
func (sm statsManifest) Exists(dgst digest.Digest) (bool, error) {
func (sm statsManifest) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
sm.stats["exists"]++
return sm.manifests.Exists(dgst)
return sm.manifests.Exists(ctx, dgst)
}
func (sm statsManifest) ExistsByTag(tag string) (bool, error) {
sm.stats["existbytag"]++
return sm.manifests.ExistsByTag(tag)
}
func (sm statsManifest) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
func (sm statsManifest) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
sm.stats["get"]++
return sm.manifests.Get(dgst)
return sm.manifests.Get(ctx, dgst)
}
func (sm statsManifest) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
sm.stats["getbytag"]++
return sm.manifests.GetByTag(tag, options...)
}
func (sm statsManifest) Put(manifest *schema1.SignedManifest) error {
func (sm statsManifest) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
sm.stats["put"]++
return sm.manifests.Put(manifest)
return sm.manifests.Put(ctx, manifest)
}
func (sm statsManifest) Tags() ([]string, error) {
sm.stats["tags"]++
return sm.manifests.Tags()
/*func (sm statsManifest) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
sm.stats["enumerate"]++
return sm.manifests.Enumerate(ctx, manifests, last)
}
*/
func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
ctx := context.Background()
@ -169,15 +160,12 @@ func populateRepo(t *testing.T, ctx context.Context, repository distribution.Rep
if err != nil {
t.Fatalf(err.Error())
}
ms.Put(sm)
dgst, err := ms.Put(ctx, sm)
if err != nil {
t.Fatalf("unexpected errors putting manifest: %v", err)
}
pl, err := sm.Payload()
if err != nil {
t.Fatal(err)
}
return digest.FromBytes(pl), nil
return dgst, nil
}
// TestProxyManifests contains basic acceptance tests
@ -189,8 +177,9 @@ func TestProxyManifests(t *testing.T) {
localStats := env.LocalStats()
remoteStats := env.RemoteStats()
ctx := context.Background()
// Stat - must check local and remote
exists, err := env.manifests.ExistsByTag("latest")
exists, err := env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatalf("Error checking existance")
}
@ -198,15 +187,16 @@ func TestProxyManifests(t *testing.T) {
t.Errorf("Unexpected non-existant manifest")
}
if (*localStats)["existbytag"] != 1 && (*remoteStats)["existbytag"] != 1 {
t.Errorf("Unexpected exists count")
if (*localStats)["exists"] != 1 && (*remoteStats)["exists"] != 1 {
t.Errorf("Unexpected exists count : \n%v \n%v", localStats, remoteStats)
}
// Get - should succeed and pull manifest into local
_, err = env.manifests.Get(env.manifestDigest)
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if (*localStats)["get"] != 1 && (*remoteStats)["get"] != 1 {
t.Errorf("Unexpected get count")
}
@ -216,7 +206,7 @@ func TestProxyManifests(t *testing.T) {
}
// Stat - should only go to local
exists, err = env.manifests.ExistsByTag("latest")
exists, err = env.manifests.Exists(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
@ -224,19 +214,21 @@ func TestProxyManifests(t *testing.T) {
t.Errorf("Unexpected non-existant manifest")
}
if (*localStats)["existbytag"] != 2 && (*remoteStats)["existbytag"] != 1 {
if (*localStats)["exists"] != 2 && (*remoteStats)["exists"] != 1 {
t.Errorf("Unexpected exists count")
}
// Get - should get from remote, to test freshness
_, err = env.manifests.Get(env.manifestDigest)
_, err = env.manifests.Get(ctx, env.manifestDigest)
if err != nil {
t.Fatal(err)
}
if (*remoteStats)["get"] != 2 && (*remoteStats)["existsbytag"] != 1 && (*localStats)["put"] != 1 {
if (*remoteStats)["get"] != 2 && (*remoteStats)["exists"] != 1 && (*localStats)["put"] != 1 {
t.Errorf("Unexpected get count")
}
}
func TestProxyTagService(t *testing.T) {
}

23
docs/proxy/proxyregistry.go
View file

@ -42,6 +42,7 @@ func NewRegistryPullThroughCache(ctx context.Context, registry distribution.Name
s.OnManifestExpire(func(repoName string) error {
return v.RemoveRepository(repoName)
})
err = s.Start()
if err != nil {
return nil, err
@ -78,7 +79,7 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
if err != nil {
return nil, err
}
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification)
localManifests, err := localRepo.Manifests(ctx, storage.SkipLayerVerification())
if err != nil {
return nil, err
}
@ -106,8 +107,11 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
ctx: ctx,
scheduler: pr.scheduler,
},
name: name,
signatures: localRepo.Signatures(),
name: name,
tags: proxyTagService{
localTags: localRepo.Tags(ctx),
remoteTags: remoteRepo.Tags(ctx),
},
}, nil
}
@ -115,14 +119,13 @@ func (pr *proxyingRegistry) Repository(ctx context.Context, name string) (distri
// locally, or pulling it through from a remote and caching it locally if it doesn't
// already exist
type proxiedRepository struct {
blobStore distribution.BlobStore
manifests distribution.ManifestService
name string
signatures distribution.SignatureService
blobStore distribution.BlobStore
manifests distribution.ManifestService
name string
tags distribution.TagService
}
func (pr *proxiedRepository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
// options
return pr.manifests, nil
}
@ -134,6 +137,6 @@ func (pr *proxiedRepository) Name() string {
return pr.name
}
func (pr *proxiedRepository) Signatures() distribution.SignatureService {
return pr.signatures
func (pr *proxiedRepository) Tags(ctx context.Context) distribution.TagService {
return pr.tags
}

58
docs/proxy/proxytagservice.go Normal file
View file

@ -0,0 +1,58 @@
package proxy
import (
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
// proxyTagService supports local and remote lookup of tags.
type proxyTagService struct {
localTags distribution.TagService
remoteTags distribution.TagService
}
var _ distribution.TagService = proxyTagService{}
// Get attempts to get the most recent digest for the tag by checking the remote
// tag service first and then caching it locally. If the remote is unavailable
// the local association is returned
func (pt proxyTagService) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
desc, err := pt.remoteTags.Get(ctx, tag)
if err == nil {
err := pt.localTags.Tag(ctx, tag, desc)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
desc, err = pt.localTags.Get(ctx, tag)
if err != nil {
return distribution.Descriptor{}, err
}
return desc, nil
}
func (pt proxyTagService) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
return distribution.ErrUnsupported
}
func (pt proxyTagService) Untag(ctx context.Context, tag string) error {
err := pt.localTags.Untag(ctx, tag)
if err != nil {
return err
}
return nil
}
func (pt proxyTagService) All(ctx context.Context) ([]string, error) {
tags, err := pt.remoteTags.All(ctx)
if err == nil {
return tags, err
}
return pt.localTags.All(ctx)
}
func (pt proxyTagService) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
return []string{}, distribution.ErrUnsupported
}

164
docs/proxy/proxytagservice_test.go Normal file
View file

@ -0,0 +1,164 @@
package proxy
import (
"sort"
"sync"
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
)
type mockTagStore struct {
mapping map[string]distribution.Descriptor
sync.Mutex
}
var _ distribution.TagService = &mockTagStore{}
func (m *mockTagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
m.Lock()
defer m.Unlock()
if d, ok := m.mapping[tag]; ok {
return d, nil
}
return distribution.Descriptor{}, distribution.ErrTagUnknown{}
}
func (m *mockTagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
m.Lock()
defer m.Unlock()
m.mapping[tag] = desc
return nil
}
func (m *mockTagStore) Untag(ctx context.Context, tag string) error {
m.Lock()
defer m.Unlock()
if _, ok := m.mapping[tag]; ok {
delete(m.mapping, tag)
return nil
}
return distribution.ErrTagUnknown{}
}
func (m *mockTagStore) All(ctx context.Context) ([]string, error) {
m.Lock()
defer m.Unlock()
var tags []string
for tag := range m.mapping {
tags = append(tags, tag)
}
return tags, nil
}
func (m *mockTagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
panic("not implemented")
}
func testProxyTagService(local, remote map[string]distribution.Descriptor) *proxyTagService {
if local == nil {
local = make(map[string]distribution.Descriptor)
}
if remote == nil {
remote = make(map[string]distribution.Descriptor)
}
return &proxyTagService{
localTags: &mockTagStore{mapping: local},
remoteTags: &mockTagStore{mapping: remote},
}
}
func TestGet(t *testing.T) {
remoteDesc := distribution.Descriptor{Size: 42}
remoteTag := "remote"
proxyTags := testProxyTagService(map[string]distribution.Descriptor{remoteTag: remoteDesc}, nil)
ctx := context.Background()
// Get pre-loaded tag
d, err := proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
if d != remoteDesc {
t.Fatal("unable to get put tag")
}
local, err := proxyTags.localTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("remote tag not pulled into store")
}
if local != remoteDesc {
t.Fatalf("unexpected descriptor pulled through")
}
// Manually overwrite remote tag
newRemoteDesc := distribution.Descriptor{Size: 43}
err = proxyTags.remoteTags.Tag(ctx, remoteTag, newRemoteDesc)
if err != nil {
t.Fatal(err)
}
d, err = proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
if d != newRemoteDesc {
t.Fatal("unable to get put tag")
}
_, err = proxyTags.localTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("remote tag not pulled into store")
}
// untag, ensure it's removed locally, but present in remote
err = proxyTags.Untag(ctx, remoteTag)
if err != nil {
t.Fatal(err)
}
_, err = proxyTags.localTags.Get(ctx, remoteTag)
if err == nil {
t.Fatalf("Expected error getting Untag'd tag")
}
_, err = proxyTags.remoteTags.Get(ctx, remoteTag)
if err != nil {
t.Fatalf("remote tag should not be untagged with proxyTag.Untag")
}
_, err = proxyTags.Get(ctx, remoteTag)
if err != nil {
t.Fatal("untagged tag should be pulled through")
}
// Add another tag. Ensure both tags appear in enumerate
err = proxyTags.remoteTags.Tag(ctx, "funtag", distribution.Descriptor{Size: 42})
if err != nil {
t.Fatal(err)
}
all, err := proxyTags.All(ctx)
if err != nil {
t.Fatal(err)
}
if len(all) != 2 {
t.Fatalf("Unexpected tag length returned from All() : %d ", len(all))
}
sort.Strings(all)
if all[0] != "funtag" && all[1] != "remote" {
t.Fatalf("Unexpected tags returned from All() : %v ", all)
}
}

159
docs/storage/manifeststore.go
View file

@ -1,6 +1,7 @@
package storage
import (
"encoding/json"
"fmt"
"github.com/docker/distribution"
@ -11,20 +12,21 @@ import (
"github.com/docker/libtrust"
)
// manifestStore is a storage driver based store for storing schema1 manifests.
type manifestStore struct {
repository *repository
revisionStore *revisionStore
tagStore *tagStore
blobStore *linkedBlobStore
ctx context.Context
signatures *signatureStore
skipDependencyVerification bool
}
var _ distribution.ManifestService = &manifestStore{}
func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
func (ms *manifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists")
_, err := ms.revisionStore.blobStore.Stat(ms.ctx, dgst)
_, err := ms.blobStore.Stat(ms.ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return false, nil
@ -36,76 +38,131 @@ func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) {
return true, nil
}
func (ms *manifestStore) Get(dgst digest.Digest) (*schema1.SignedManifest, error) {
func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Get")
return ms.revisionStore.get(ms.ctx, dgst)
// Ensure that this revision is available in this repository.
_, err := ms.blobStore.Stat(ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: ms.repository.Name(),
Revision: dgst,
}
}
return nil, err
}
// TODO(stevvooe): Need to check descriptor from above to ensure that the
// mediatype is as we expect for the manifest store.
content, err := ms.blobStore.Get(ctx, dgst)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: ms.repository.Name(),
Revision: dgst,
}
}
return nil, err
}
// Fetch the signatures for the manifest
signatures, err := ms.signatures.Get(dgst)
if err != nil {
return nil, err
}
jsig, err := libtrust.NewJSONSignature(content, signatures...)
if err != nil {
return nil, err
}
// Extract the pretty JWS
raw, err := jsig.PrettySignature("signatures")
if err != nil {
return nil, err
}
var sm schema1.SignedManifest
if err := json.Unmarshal(raw, &sm); err != nil {
return nil, err
}
return &sm, nil
}
// SkipLayerVerification allows a manifest to be Put before it's
// SkipLayerVerification allows a manifest to be Put before its
// layers are on the filesystem
func SkipLayerVerification(ms distribution.ManifestService) error {
if ms, ok := ms.(*manifestStore); ok {
func SkipLayerVerification() distribution.ManifestServiceOption {
return skipLayerOption{}
}
type skipLayerOption struct{}
func (o skipLayerOption) Apply(m distribution.ManifestService) error {
if ms, ok := m.(*manifestStore); ok {
ms.skipDependencyVerification = true
return nil
}
return fmt.Errorf("skip layer verification only valid for manifestStore")
}
func (ms *manifestStore) Put(manifest *schema1.SignedManifest) error {
func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Put")
if err := ms.verifyManifest(ms.ctx, manifest); err != nil {
return err
sm, ok := manifest.(*schema1.SignedManifest)
if !ok {
return "", fmt.Errorf("non-v1 manifest put to signed manifestStore: %T", manifest)
}
// Store the revision of the manifest
revision, err := ms.revisionStore.put(ms.ctx, manifest)
if err := ms.verifyManifest(ms.ctx, *sm); err != nil {
return "", err
}
mt := schema1.MediaTypeManifest
payload := sm.Canonical
revision, err := ms.blobStore.Put(ctx, mt, payload)
if err != nil {
return err
context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
return "", err
}
// Now, tag the manifest
return ms.tagStore.tag(manifest.Tag, revision.Digest)
// Link the revision into the repository.
if err := ms.blobStore.linkBlob(ctx, revision); err != nil {
return "", err
}
// Grab each json signature and store them.
signatures, err := sm.Signatures()
if err != nil {
return "", err
}
if err := ms.signatures.Put(revision.Digest, signatures...); err != nil {
return "", err
}
return revision.Digest, nil
}
// Delete removes the revision of the specified manfiest.
func (ms *manifestStore) Delete(dgst digest.Digest) error {
func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete")
return ms.revisionStore.delete(ms.ctx, dgst)
return ms.blobStore.Delete(ctx, dgst)
}
func (ms *manifestStore) Tags() ([]string, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).Tags")
return ms.tagStore.tags()
}
func (ms *manifestStore) ExistsByTag(tag string) (bool, error) {
context.GetLogger(ms.ctx).Debug("(*manifestStore).ExistsByTag")
return ms.tagStore.exists(tag)
}
func (ms *manifestStore) GetByTag(tag string, options ...distribution.ManifestServiceOption) (*schema1.SignedManifest, error) {
for _, option := range options {
err := option(ms)
if err != nil {
return nil, err
}
}
context.GetLogger(ms.ctx).Debug("(*manifestStore).GetByTag")
dgst, err := ms.tagStore.resolve(tag)
if err != nil {
return nil, err
}
return ms.revisionStore.get(ms.ctx, dgst)
func (ms *manifestStore) Enumerate(ctx context.Context, manifests []distribution.Manifest, last distribution.Manifest) (n int, err error) {
return 0, distribution.ErrUnsupported
}
// verifyManifest ensures that the manifest content is valid from the
// perspective of the registry. It ensures that the signature is valid for the
// enclosed payload. As a policy, the registry only tries to store valid
// content, leaving trust policies of that content up to consumers.
func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.SignedManifest) error {
// content, leaving trust policies of that content up to consumems.
func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest) error {
var errs distribution.ErrManifestVerification
if len(mnfst.Name) > reference.NameTotalLengthMax {
@ -129,7 +186,7 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign
len(mnfst.History), len(mnfst.FSLayers)))
}
if _, err := schema1.Verify(mnfst); err != nil {
if _, err := schema1.Verify(&mnfst); err != nil {
switch err {
case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
errs = append(errs, distribution.ErrManifestUnverified{})
@ -143,15 +200,15 @@ func (ms *manifestStore) verifyManifest(ctx context.Context, mnfst *schema1.Sign
}
if !ms.skipDependencyVerification {
for _, fsLayer := range mnfst.FSLayers {
_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.BlobSum)
for _, fsLayer := range mnfst.References() {
_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest)
if err != nil {
if err != distribution.ErrBlobUnknown {
errs = append(errs, err)
}
// On error here, we always append unknown blob errors.
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.BlobSum})
// On error here, we always append unknown blob erroms.
errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest})
}
}
}

100
docs/storage/manifeststore_test.go
View file

@ -30,7 +30,8 @@ type manifestStoreTestEnv struct {
func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
ctx := context.Background()
driver := inmemory.New()
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(
memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
@ -58,24 +59,6 @@ func TestManifestStorage(t *testing.T) {
t.Fatal(err)
}
exists, err := ms.ExistsByTag(env.tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if exists {
t.Fatalf("manifest should not exist")
}
if _, err := ms.GetByTag(env.tag); true {
switch err.(type) {
case distribution.ErrManifestUnknown:
break
default:
t.Fatalf("expected manifest unknown error: %#v", err)
}
}
m := schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
@ -114,7 +97,7 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("error signing manifest: %v", err)
}
err = ms.Put(sm)
_, err = ms.Put(ctx, sm)
if err == nil {
t.Fatalf("expected errors putting manifest with full verification")
}
@ -150,30 +133,40 @@ func TestManifestStorage(t *testing.T) {
}
}
if err = ms.Put(sm); err != nil {
var manifestDigest digest.Digest
if manifestDigest, err = ms.Put(ctx, sm); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
exists, err = ms.ExistsByTag(env.tag)
exists, err := ms.Exists(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
t.Fatalf("unexpected error checking manifest existence: %#v", err)
}
if !exists {
t.Fatalf("manifest should exist")
}
fetchedManifest, err := ms.GetByTag(env.tag)
fromStore, err := ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
fetchedManifest, ok := fromStore.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected manifest type from signedstore")
}
if !reflect.DeepEqual(fetchedManifest, sm) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm)
}
fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures")
_, pl, err := fetchedManifest.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
fetchedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
@ -185,8 +178,9 @@ func TestManifestStorage(t *testing.T) {
// Now that we have a payload, take a moment to check that the manifest is
// return by the payload digest.
dgst := digest.FromBytes(payload)
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("error checking manifest existence by digest: %v", err)
}
@ -195,7 +189,7 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("manifest %s should exist", dgst)
}
fetchedByDigest, err := ms.Get(dgst)
fetchedByDigest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest by digest: %v", err)
}
@ -213,20 +207,6 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1)
}
// Grabs the tags and check that this tagged manifest is present
tags, err := ms.Tags()
if err != nil {
t.Fatalf("unexpected error fetching tags: %v", err)
}
if len(tags) != 1 {
t.Fatalf("unexpected tags returned: %v", tags)
}
if tags[0] != env.tag {
t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag})
}
// Now, push the same manifest with a different key
pk2, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
@ -237,8 +217,12 @@ func TestManifestStorage(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
_, pl, err = sm2.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures")
jws2, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("error parsing signature: %v", err)
}
@ -252,15 +236,20 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1)
}
if err = ms.Put(sm2); err != nil {
if manifestDigest, err = ms.Put(ctx, sm2); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
fetched, err := ms.GetByTag(env.tag)
fromStore, err = ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
fetched, ok := fromStore.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected type from signed manifeststore : %T", fetched)
}
if _, err := schema1.Verify(fetched); err != nil {
t.Fatalf("unexpected error verifying manifest: %v", err)
}
@ -276,7 +265,12 @@ func TestManifestStorage(t *testing.T) {
t.Fatalf("unexpected error getting expected signatures: %v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures")
_, pl, err = fetched.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
@ -302,12 +296,12 @@ func TestManifestStorage(t *testing.T) {
}
// Test deleting manifests
err = ms.Delete(dgst)
err = ms.Delete(ctx, dgst)
if err != nil {
t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
}
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
@ -315,7 +309,7 @@ func TestManifestStorage(t *testing.T) {
t.Errorf("Deleted manifest should not exist")
}
deletedManifest, err := ms.Get(dgst)
deletedManifest, err := ms.Get(ctx, dgst)
if err == nil {
t.Errorf("Unexpected success getting deleted manifest")
}
@ -331,12 +325,12 @@ func TestManifestStorage(t *testing.T) {
}
// Re-upload should restore manifest to a good state
err = ms.Put(sm)
_, err = ms.Put(ctx, sm)
if err != nil {
t.Errorf("Error re-uploading deleted manifest")
}
exists, err = ms.Exists(dgst)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
@ -344,7 +338,7 @@ func TestManifestStorage(t *testing.T) {
t.Errorf("Restored manifest should exist")
}
deletedManifest, err = ms.Get(dgst)
deletedManifest, err = ms.Get(ctx, dgst)
if err != nil {
t.Errorf("Unexpected error getting manifest")
}
@ -364,7 +358,7 @@ func TestManifestStorage(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = ms.Delete(dgst)
err = ms.Delete(ctx, dgst)
if err == nil {
t.Errorf("Unexpected success deleting while disabled")
}

54
docs/storage/registry.go
View file

@ -145,6 +145,15 @@ func (repo *repository) Name() string {
return repo.name
}
func (repo *repository) Tags(ctx context.Context) distribution.TagService {
tags := &tagStore{
repository: repo,
blobStore: repo.registry.blobStore,
}
return tags
}
// Manifests returns an instance of ManifestService. Instantiation is cheap and
// may be context sensitive in the future. The instance should be used similar
// to a request local.
@ -159,36 +168,31 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M
ms := &manifestStore{
ctx: ctx,
repository: repo,
revisionStore: &revisionStore{
ctx: ctx,
repository: repo,
blobStore: &linkedBlobStore{
ctx: ctx,
blobStore: repo.blobStore,
repository: repo,
deleteEnabled: repo.registry.deleteEnabled,
blobAccessController: &linkedBlobStatter{
blobStore: repo.blobStore,
repository: repo,
linkPathFns: manifestLinkPathFns,
},
// TODO(stevvooe): linkPath limits this blob store to only
// manifests. This instance cannot be used for blob checks.
linkPathFns: manifestLinkPathFns,
resumableDigestEnabled: repo.resumableDigestEnabled,
blobStore: &linkedBlobStore{
ctx: ctx,
blobStore: repo.blobStore,
repository: repo,
deleteEnabled: repo.registry.deleteEnabled,
blobAccessController: &linkedBlobStatter{
blobStore: repo.blobStore,
repository: repo,
linkPathFns: manifestLinkPathFns,
},
// TODO(stevvooe): linkPath limits this blob store to only
// manifests. This instance cannot be used for blob checks.
linkPathFns: manifestLinkPathFns,
},
tagStore: &tagStore{
signatures: &signatureStore{
ctx: ctx,
repository: repo,
blobStore: repo.registry.blobStore,
blobStore: repo.blobStore,
},
}
// Apply options
for _, option := range options {
err := option(ms)
err := option.Apply(ms)
if err != nil {
return nil, err
}
@ -225,11 +229,3 @@ func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore {
resumableDigestEnabled: repo.resumableDigestEnabled,
}
}
func (repo *repository) Signatures() distribution.SignatureService {
return &signatureStore{
repository: repo,
blobStore: repo.blobStore,
ctx: repo.ctx,
}
}

111
docs/storage/revisionstore.go
View file

@ -1,111 +0,0 @@
package storage
import (
"encoding/json"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/libtrust"
)
// revisionStore supports storing and managing manifest revisions.
type revisionStore struct {
repository *repository
blobStore *linkedBlobStore
ctx context.Context
}
// get retrieves the manifest, keyed by revision digest.
func (rs *revisionStore) get(ctx context.Context, revision digest.Digest) (*schema1.SignedManifest, error) {
// Ensure that this revision is available in this repository.
_, err := rs.blobStore.Stat(ctx, revision)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: rs.repository.Name(),
Revision: revision,
}
}
return nil, err
}
// TODO(stevvooe): Need to check descriptor from above to ensure that the
// mediatype is as we expect for the manifest store.
content, err := rs.blobStore.Get(ctx, revision)
if err != nil {
if err == distribution.ErrBlobUnknown {
return nil, distribution.ErrManifestUnknownRevision{
Name: rs.repository.Name(),
Revision: revision,
}
}
return nil, err
}
// Fetch the signatures for the manifest
signatures, err := rs.repository.Signatures().Get(revision)
if err != nil {
return nil, err
}
jsig, err := libtrust.NewJSONSignature(content, signatures...)
if err != nil {
return nil, err
}
// Extract the pretty JWS
raw, err := jsig.PrettySignature("signatures")
if err != nil {
return nil, err
}
var sm schema1.SignedManifest
if err := json.Unmarshal(raw, &sm); err != nil {
return nil, err
}
return &sm, nil
}
// put stores the manifest in the repository, if not already present. Any
// updated signatures will be stored, as well.
func (rs *revisionStore) put(ctx context.Context, sm *schema1.SignedManifest) (distribution.Descriptor, error) {
// Resolve the payload in the manifest.
payload, err := sm.Payload()
if err != nil {
return distribution.Descriptor{}, err
}
// Digest and store the manifest payload in the blob store.
revision, err := rs.blobStore.Put(ctx, schema1.ManifestMediaType, payload)
if err != nil {
context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
return distribution.Descriptor{}, err
}
// Link the revision into the repository.
if err := rs.blobStore.linkBlob(ctx, revision); err != nil {
return distribution.Descriptor{}, err
}
// Grab each json signature and store them.
signatures, err := sm.Signatures()
if err != nil {
return distribution.Descriptor{}, err
}
if err := rs.repository.Signatures().Put(revision.Digest, signatures...); err != nil {
return distribution.Descriptor{}, err
}
return revision, nil
}
func (rs *revisionStore) delete(ctx context.Context, revision digest.Digest) error {
return rs.blobStore.Delete(ctx, revision)
}

11
docs/storage/signaturestore.go
View file

@ -4,7 +4,6 @@ import (
"path"
"sync"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
)
@ -15,16 +14,6 @@ type signatureStore struct {
ctx context.Context
}
func newSignatureStore(ctx context.Context, repo *repository, blobStore *blobStore) *signatureStore {
return &signatureStore{
ctx: ctx,
repository: repo,
blobStore: blobStore,
}
}
var _ distribution.SignatureService = &signatureStore{}
func (s *signatureStore) Get(dgst digest.Digest) ([][]byte, error) {
signaturesPath, err := pathFor(manifestSignaturesPathSpec{
name: s.repository.Name(),

64
docs/storage/tagstore.go
View file

@ -9,37 +9,41 @@ import (
storagedriver "github.com/docker/distribution/registry/storage/driver"
)
var _ distribution.TagService = &tagStore{}
// tagStore provides methods to manage manifest tags in a backend storage driver.
// This implementation uses the same on-disk layout as the (now deleted) tag
// store. This provides backward compatibility with current registry deployments
// which only makes use of the Digest field of the returned distribution.Descriptor
// but does not enable full roundtripping of Descriptor objects
type tagStore struct {
repository *repository
blobStore *blobStore
ctx context.Context
}
// tags lists the manifest tags for the specified repository.
func (ts *tagStore) tags() ([]string, error) {
p, err := pathFor(manifestTagPathSpec{
// All returns all tags
func (ts *tagStore) All(ctx context.Context) ([]string, error) {
var tags []string
pathSpec, err := pathFor(manifestTagPathSpec{
name: ts.repository.Name(),
})
if err != nil {
return nil, err
return tags, err
}
var tags []string
entries, err := ts.blobStore.driver.List(ts.ctx, p)
entries, err := ts.blobStore.driver.List(ctx, pathSpec)
if err != nil {
switch err := err.(type) {
case storagedriver.PathNotFoundError:
return nil, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()}
return tags, distribution.ErrRepositoryUnknown{Name: ts.repository.Name()}
default:
return nil, err
return tags, err
}
}
for _, entry := range entries {
_, filename := path.Split(entry)
tags = append(tags, filename)
}
@ -47,7 +51,7 @@ func (ts *tagStore) tags() ([]string, error) {
}
// exists returns true if the specified manifest tag exists in the repository.
func (ts *tagStore) exists(tag string) (bool, error) {
func (ts *tagStore) exists(ctx context.Context, tag string) (bool, error) {
tagPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -57,7 +61,7 @@ func (ts *tagStore) exists(tag string) (bool, error) {
return false, err
}
exists, err := exists(ts.ctx, ts.blobStore.driver, tagPath)
exists, err := exists(ctx, ts.blobStore.driver, tagPath)
if err != nil {
return false, err
}
@ -65,9 +69,9 @@ func (ts *tagStore) exists(tag string) (bool, error) {
return exists, nil
}
// tag tags the digest with the given tag, updating the the store to point at
// Tag tags the digest with the given tag, updating the the store to point at
// the current tag. The digest must point to a manifest.
func (ts *tagStore) tag(tag string, revision digest.Digest) error {
func (ts *tagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
currentPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -77,43 +81,44 @@ func (ts *tagStore) tag(tag string, revision digest.Digest) error {
return err
}
nbs := ts.linkedBlobStore(ts.ctx, tag)
lbs := ts.linkedBlobStore(ctx, tag)
// Link into the index
if err := nbs.linkBlob(ts.ctx, distribution.Descriptor{Digest: revision}); err != nil {
if err := lbs.linkBlob(ctx, desc); err != nil {
return err
}
// Overwrite the current link
return ts.blobStore.link(ts.ctx, currentPath, revision)
return ts.blobStore.link(ctx, currentPath, desc.Digest)
}
// resolve the current revision for name and tag.
func (ts *tagStore) resolve(tag string) (digest.Digest, error) {
func (ts *tagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
currentPath, err := pathFor(manifestTagCurrentPathSpec{
name: ts.repository.Name(),
tag: tag,
})
if err != nil {
return "", err
return distribution.Descriptor{}, err
}
revision, err := ts.blobStore.readlink(ts.ctx, currentPath)
revision, err := ts.blobStore.readlink(ctx, currentPath)
if err != nil {
switch err.(type) {
case storagedriver.PathNotFoundError:
return "", distribution.ErrManifestUnknown{Name: ts.repository.Name(), Tag: tag}
return distribution.Descriptor{}, distribution.ErrTagUnknown{Tag: tag}
}
return "", err
return distribution.Descriptor{}, err
}
return revision, nil
return distribution.Descriptor{Digest: revision}, nil
}
// delete removes the tag from repository, including the history of all
// revisions that have the specified tag.
func (ts *tagStore) delete(tag string) error {
func (ts *tagStore) Untag(ctx context.Context, tag string) error {
tagPath, err := pathFor(manifestTagPathSpec{
name: ts.repository.Name(),
tag: tag,
@ -123,7 +128,7 @@ func (ts *tagStore) delete(tag string) error {
return err
}
return ts.blobStore.driver.Delete(ts.ctx, tagPath)
return ts.blobStore.driver.Delete(ctx, tagPath)
}
// linkedBlobStore returns the linkedBlobStore for the named tag, allowing one
@ -145,3 +150,10 @@ func (ts *tagStore) linkedBlobStore(ctx context.Context, tag string) *linkedBlob
}},
}
}
// Lookup recovers a list of tags which refer to this digest. When a manifest is deleted by
// digest, tag entries which point to it need to be recovered to avoid dangling tags.
func (ts *tagStore) Lookup(ctx context.Context, digest distribution.Descriptor) ([]string, error) {
// An efficient implementation of this will require changes to the S3 driver.
return make([]string, 0), nil
}

150
docs/storage/tagstore_test.go Normal file
View file

@ -0,0 +1,150 @@
package storage
import (
"testing"
"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/storage/driver/inmemory"
)
type tagsTestEnv struct {
ts distribution.TagService
ctx context.Context
}
func testTagStore(t *testing.T) *tagsTestEnv {
ctx := context.Background()
d := inmemory.New()
reg, err := NewRegistry(ctx, d)
if err != nil {
t.Fatal(err)
}
repo, err := reg.Repository(ctx, "a/b")
if err != nil {
t.Fatal(err)
}
return &tagsTestEnv{
ctx: ctx,
ts: repo.Tags(ctx),
}
}
func TestTagStoreTag(t *testing.T) {
env := testTagStore(t)
tags := env.ts
ctx := env.ctx
d := distribution.Descriptor{}
err := tags.Tag(ctx, "latest", d)
if err == nil {
t.Errorf("unexpected error putting malformed descriptor : %s", err)
}
d.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
err = tags.Tag(ctx, "latest", d)
if err != nil {
t.Error(err)
}
d1, err := tags.Get(ctx, "latest")
if err != nil {
t.Error(err)
}
if d1.Digest != d.Digest {
t.Error("put and get digest differ")
}
// Overwrite existing
d.Digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
err = tags.Tag(ctx, "latest", d)
if err != nil {
t.Error(err)
}
d1, err = tags.Get(ctx, "latest")
if err != nil {
t.Error(err)
}
if d1.Digest != d.Digest {
t.Error("put and get digest differ")
}
}
func TestTagStoreUnTag(t *testing.T) {
env := testTagStore(t)
tags := env.ts
ctx := env.ctx
desc := distribution.Descriptor{Digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}
err := tags.Untag(ctx, "latest")
if err == nil {
t.Errorf("Expected error untagging non-existant tag")
}
err = tags.Tag(ctx, "latest", desc)
if err != nil {
t.Error(err)
}
err = tags.Untag(ctx, "latest")
if err != nil {
t.Error(err)
}
_, err = tags.Get(ctx, "latest")
if err == nil {
t.Error("Expected error getting untagged tag")
}
}
func TestTagAll(t *testing.T) {
env := testTagStore(t)
tagStore := env.ts
ctx := env.ctx
alpha := "abcdefghijklmnopqrstuvwxyz"
for i := 0; i < len(alpha); i++ {
tag := alpha[i]
desc := distribution.Descriptor{Digest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"}
err := tagStore.Tag(ctx, string(tag), desc)
if err != nil {
t.Error(err)
}
}
all, err := tagStore.All(ctx)
if err != nil {
t.Error(err)
}
if len(all) != len(alpha) {
t.Errorf("Unexpected count returned from enumerate")
}
for i, c := range all {
if c != string(alpha[i]) {
t.Errorf("unexpected tag in enumerate %s", c)
}
}
removed := "a"
err = tagStore.Untag(ctx, removed)
if err != nil {
t.Error(err)
}
all, err = tagStore.All(ctx)
if err != nil {
t.Error(err)
}
for _, tag := range all {
if tag == removed {
t.Errorf("unexpected tag in enumerate %s", removed)
}
}
}