Add support for layers from foreign sources

This will be used to support downloading Windows base layers from
Microsoft URLs.

Signed-off-by: John Starks <jostarks@microsoft.com>

blobs.go

@@ -69,6 +69,9 @@ type Descriptor struct {
 	// against against this digest.
 	Digest digest.Digest `json:"digest,omitempty"`
 
+	// URLs contains the source URLs of this content.
+	URLs []string `json:"urls,omitempty"`
+
 	// NOTE: Before adding a field here, please ensure that all
 	// other options have been exhausted. Much of the type relationships
 	// depend on the simplicity of this type.

docs/spec/manifest-v2-2.md

@@ -216,6 +216,14 @@ image. It's the direct replacement for the schema-1 manifest.
         The digest of the content, as defined by the
         [Registry V2 HTTP API Specificiation](https://docs.docker.com/registry/spec/api/#digest-parameter).
 
+    - **`urls`** *array*
+
+        For an ordinary layer, this is empty, and the layer contents can be
+        retrieved directly from the registry. For a layer with *`mediatype`* of
+        `application/vnd.docker.image.rootfs.foreign.diff.tar.gzip`, this
+        contains a non-empty list of URLs from which this object can be
+        downloaded.
+
 ## Example Image Manifest
 
 *Example showing an image manifest:*

manifest/schema2/manifest.go

@@ -20,6 +20,10 @@ const (
 	// MediaTypeLayer is the mediaType used for layers referenced by the
 	// manifest.
 	MediaTypeLayer = "application/vnd.docker.image.rootfs.diff.tar.gzip"
+
+	// MediaTypeForeignLayer is the mediaType used for layers that must be
+	// downloaded from foreign URLs.
+	MediaTypeForeignLayer = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
 )
 
 var (
@@ -63,7 +67,6 @@ type Manifest struct {
 // References returnes the descriptors of this manifests references.
 func (m Manifest) References() []distribution.Descriptor {
 	return m.Layers
-
 }
 
 // Target returns the target of this signed manifest.

registry/proxy/proxytagservice_test.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"reflect"
 	"sort"
 	"sync"
 	"testing"
@@ -92,7 +93,7 @@ func TestGet(t *testing.T) {
 		t.Fatalf("Expected 1 auth challenge call, got %#v", proxyTags.authChallenger)
 	}
 
-	if d != remoteDesc {
+	if !reflect.DeepEqual(d, remoteDesc) {
 		t.Fatal("unable to get put tag")
 	}
 
@@ -101,7 +102,7 @@ func TestGet(t *testing.T) {
 		t.Fatal("remote tag not pulled into store")
 	}
 
-	if local != remoteDesc {
+	if !reflect.DeepEqual(local, remoteDesc) {
 		t.Fatalf("unexpected descriptor pulled through")
 	}
 
@@ -121,7 +122,7 @@ func TestGet(t *testing.T) {
 		t.Fatalf("Expected 2 auth challenge calls, got %#v", proxyTags.authChallenger)
 	}
 
-	if d != newRemoteDesc {
+	if !reflect.DeepEqual(d, newRemoteDesc) {
 		t.Fatal("unable to get put tag")
 	}
 

registry/storage/blob_test.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
+	"reflect"
 	"testing"
 
 	"github.com/docker/distribution"
@@ -16,7 +18,6 @@ import (
 	"github.com/docker/distribution/registry/storage/cache/memory"
 	"github.com/docker/distribution/registry/storage/driver/inmemory"
 	"github.com/docker/distribution/testutil"
-	"path"
 )
 
 // TestWriteSeek tests that the current file size can be
@@ -156,7 +157,7 @@ func TestSimpleBlobUpload(t *testing.T) {
 		t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
 	}
 
-	if statDesc != desc {
+	if !reflect.DeepEqual(statDesc, desc) {
 		t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
 	}
 
@@ -410,7 +411,7 @@ func TestBlobMount(t *testing.T) {
 		t.Fatalf("unexpected error checking for existence: %v, %#v", err, sbs)
 	}
 
-	if statDesc != desc {
+	if !reflect.DeepEqual(statDesc, desc) {
 		t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
 	}
 
@@ -436,7 +437,7 @@ func TestBlobMount(t *testing.T) {
 		t.Fatalf("unexpected error mounting layer: %v", err)
 	}
 
-	if ebm.Descriptor != desc {
+	if !reflect.DeepEqual(ebm.Descriptor, desc) {
 		t.Fatalf("descriptors not equal: %v != %v", ebm.Descriptor, desc)
 	}
 
@@ -446,7 +447,7 @@ func TestBlobMount(t *testing.T) {
 		t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
 	}
 
-	if statDesc != desc {
+	if !reflect.DeepEqual(statDesc, desc) {
 		t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
 	}
 

registry/storage/cache/cachecheck/suite.go

@@ -1,6 +1,7 @@
 package cachecheck
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/docker/distribution"
@@ -79,7 +80,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
 		t.Fatalf("unexpected error statting fake2:abc: %v", err)
 	}
 
-	if expected != desc {
+	if !reflect.DeepEqual(expected, desc) {
 		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
 	}
 
@@ -89,7 +90,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
 		t.Fatalf("descriptor not returned for canonical key: %v", err)
 	}
 
-	if expected != desc {
+	if !reflect.DeepEqual(expected, desc) {
 		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
 	}
 
@@ -99,7 +100,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
 		t.Fatalf("expected blob unknown in global cache: %v, %v", err, desc)
 	}
 
-	if desc != expected {
+	if !reflect.DeepEqual(desc, expected) {
 		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
 	}
 
@@ -109,7 +110,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
 		t.Fatalf("unexpected error checking glboal descriptor: %v", err)
 	}
 
-	if desc != expected {
+	if !reflect.DeepEqual(desc, expected) {
 		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
 	}
 
@@ -126,7 +127,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
 		t.Fatalf("unexpected error getting descriptor: %v", err)
 	}
 
-	if desc != expected {
+	if !reflect.DeepEqual(desc, expected) {
 		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
 	}
 
@@ -137,7 +138,7 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
 
 	expected.MediaType = "application/octet-stream" // expect original mediatype in global
 
-	if desc != expected {
+	if !reflect.DeepEqual(desc, expected) {
 		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
 	}
 }
@@ -163,7 +164,7 @@ func checkBlobDescriptorCacheClear(t *testing.T, ctx context.Context, provider c
 		t.Fatalf("unexpected error statting fake2:abc: %v", err)
 	}
 
-	if expected != desc {
+	if !reflect.DeepEqual(expected, desc) {
 		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
 	}
 

registry/storage/schema2manifesthandler.go

@@ -1,15 +1,24 @@
 package storage
 
 import (
+	"errors"
 	"fmt"
+	"net/url"
 
 	"encoding/json"
+
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/context"
 	"github.com/docker/distribution/digest"
 	"github.com/docker/distribution/manifest/schema2"
 )
 
+var (
+	errUnexpectedURL = errors.New("unexpected URL on layer")
+	errMissingURL    = errors.New("missing URL on layer")
+	errInvalidURL    = errors.New("invalid URL on layer")
+)
+
 //schema2ManifestHandler is a ManifestHandler that covers schema2 manifests.
 type schema2ManifestHandler struct {
 	repository *repository
@@ -80,7 +89,27 @@ func (ms *schema2ManifestHandler) verifyManifest(ctx context.Context, mnfst sche
 		}
 
 		for _, fsLayer := range mnfst.References() {
-			_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest)
+			var err error
+			if fsLayer.MediaType != schema2.MediaTypeForeignLayer {
+				if len(fsLayer.URLs) == 0 {
+					_, err = ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest)
+				} else {
+					err = errUnexpectedURL
+				}
+			} else {
+				// Clients download this layer from an external URL, so do not check for
+				// its presense.
+				if len(fsLayer.URLs) == 0 {
+					err = errMissingURL
+				}
+				for _, u := range fsLayer.URLs {
+					var pu *url.URL
+					pu, err = url.Parse(u)
+					if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" {
+						err = errInvalidURL
+					}
+				}
+			}
 			if err != nil {
 				if err != distribution.ErrBlobUnknown {
 					errs = append(errs, err)

registry/storage/schema2manifesthandler_test.go

@@ -0,0 +1,117 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+)
+
+func TestVerifyManifestForeignLayer(t *testing.T) {
+	ctx := context.Background()
+	inmemoryDriver := inmemory.New()
+	registry := createRegistry(t, inmemoryDriver)
+	repo := makeRepository(t, registry, "test")
+	manifestService := makeManifestService(t, repo)
+
+	config, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeConfig, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeLayer, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	foreignLayer := distribution.Descriptor{
+		Digest:    "sha256:463435349086340864309863409683460843608348608934092322395278926a",
+		Size:      6323,
+		MediaType: schema2.MediaTypeForeignLayer,
+	}
+
+	template := schema2.Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 2,
+			MediaType:     schema2.MediaTypeManifest,
+		},
+		Config: config,
+	}
+
+	type testcase struct {
+		BaseLayer distribution.Descriptor
+		URLs      []string
+		Err       error
+	}
+
+	cases := []testcase{
+		{
+			foreignLayer,
+			nil,
+			errMissingURL,
+		},
+		{
+			layer,
+			[]string{"http://foo/bar"},
+			errUnexpectedURL,
+		},
+		{
+			foreignLayer,
+			[]string{"file:///local/file"},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"http://foo/bar#baz"},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{""},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"https://foo/bar", ""},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"http://foo/bar"},
+			nil,
+		},
+		{
+			foreignLayer,
+			[]string{"https://foo/bar"},
+			nil,
+		},
+	}
+
+	for _, c := range cases {
+		m := template
+		l := c.BaseLayer
+		l.URLs = c.URLs
+		m.Layers = []distribution.Descriptor{l}
+		dm, err := schema2.FromStruct(m)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		_, err = manifestService.Put(ctx, dm)
+		if verr, ok := err.(distribution.ErrManifestVerification); ok {
+			// Extract the first error
+			if len(verr) == 2 {
+				if _, ok = verr[1].(distribution.ErrManifestBlobUnknown); ok {
+					err = verr[0]
+				}
+			}
+		}
+		if err != c.Err {
+			t.Errorf("%#v: expected %v, got %v", l, c.Err, err)
+		}
+	}
+}