Add pull through cache functionality to the Registry which can be configured

with a new `proxy` section in the configuration file.

Create a new registry type which delegates storage to a proxyBlobStore
and proxyManifestStore.  These stores will pull through data if not present
locally.  proxyBlobStore takes care not to write duplicate data to disk.

Add a scheduler to cleanup expired content. The scheduler runs as a background
goroutine.  When a blob or manifest is pulled through from the remote registry,
an entry is added to the scheduler with a TTL.  When the TTL expires the
scheduler calls a pre-specified function to remove the fetched resource.

Add token authentication to the registry middleware.  Get a token at startup
and preload the credential store with the username and password supplied in the
config file.

Allow resumable digest functionality to be disabled at runtime and disable
it when the registry is a pull through cache.

Signed-off-by: Richard Scothern <richard.scothern@gmail.com>

docs/storage/blob_test.go

@@ -33,7 +33,7 @@ func TestSimpleBlobUpload(t *testing.T) {
 	ctx := context.Background()
 	imageName := "foo/bar"
 	driver := inmemory.New()
-	registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), true, true)
+	registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), true, true, false)
 	repository, err := registry.Repository(ctx, imageName)
 	if err != nil {
 		t.Fatalf("unexpected error getting repo: %v", err)
@@ -193,7 +193,7 @@ func TestSimpleBlobUpload(t *testing.T) {
 	}
 
 	// Reuse state to test delete with a delete-disabled registry
-	registry = NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), false, true)
+	registry = NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), false, true, false)
 	repository, err = registry.Repository(ctx, imageName)
 	if err != nil {
 		t.Fatalf("unexpected error getting repo: %v", err)
@@ -212,7 +212,7 @@ func TestSimpleBlobRead(t *testing.T) {
 	ctx := context.Background()
 	imageName := "foo/bar"
 	driver := inmemory.New()
-	registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), true, true)
+	registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), true, true, false)
 	repository, err := registry.Repository(ctx, imageName)
 	if err != nil {
 		t.Fatalf("unexpected error getting repo: %v", err)
@@ -316,7 +316,7 @@ func TestLayerUploadZeroLength(t *testing.T) {
 	ctx := context.Background()
 	imageName := "foo/bar"
 	driver := inmemory.New()
-	registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), true, true)
+	registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), true, true, false)
 	repository, err := registry.Repository(ctx, imageName)
 	if err != nil {
 		t.Fatalf("unexpected error getting repo: %v", err)

docs/storage/blobwriter.go

@@ -31,6 +31,8 @@ type blobWriter struct {
 	// implementes io.WriteSeeker, io.ReaderFrom and io.Closer to satisfy
 	// LayerUpload Interface
 	bufferedFileWriter
+
+	resumableDigestEnabled bool
 }
 
 var _ distribution.BlobWriter = &blobWriter{}
@@ -349,3 +351,29 @@ func (bw *blobWriter) removeResources(ctx context.Context) error {
 
 	return nil
 }
+
+func (bw *blobWriter) Reader() (io.ReadCloser, error) {
+	// todo(richardscothern): Change to exponential backoff, i=0.5, e=2, n=4
+	try := 1
+	for try <= 5 {
+		_, err := bw.bufferedFileWriter.driver.Stat(bw.ctx, bw.path)
+		if err == nil {
+			break
+		}
+		switch err.(type) {
+		case storagedriver.PathNotFoundError:
+			context.GetLogger(bw.ctx).Debugf("Nothing found on try %d, sleeping...", try)
+			time.Sleep(1 * time.Second)
+			try++
+		default:
+			return nil, err
+		}
+	}
+
+	readCloser, err := bw.bufferedFileWriter.driver.ReadStream(bw.ctx, bw.path, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	return readCloser, nil
+}

docs/storage/blobwriter_resumable.go

@@ -24,6 +24,10 @@ import (
 // offset. Any unhashed bytes remaining less than the given offset are hashed
 // from the content uploaded so far.
 func (bw *blobWriter) resumeDigestAt(ctx context.Context, offset int64) error {
+	if !bw.resumableDigestEnabled {
+		return errResumableDigestNotAvailable
+	}
+
 	if offset < 0 {
 		return fmt.Errorf("cannot resume hash at negative offset: %d", offset)
 	}
@@ -143,6 +147,10 @@ func (bw *blobWriter) getStoredHashStates(ctx context.Context) ([]hashStateEntry
 }
 
 func (bw *blobWriter) storeHashState(ctx context.Context) error {
+	if !bw.resumableDigestEnabled {
+		return errResumableDigestNotAvailable
+	}
+
 	h, ok := bw.digester.Hash().(resumable.Hash)
 	if !ok {
 		return errResumableDigestNotAvailable

docs/storage/catalog_test.go

@@ -22,7 +22,7 @@ func setupFS(t *testing.T) *setupEnv {
 	d := inmemory.New()
 	c := []byte("")
 	ctx := context.Background()
-	registry := NewRegistryWithDriver(ctx, d, memory.NewInMemoryBlobDescriptorCacheProvider(), false, true)
+	registry := NewRegistryWithDriver(ctx, d, memory.NewInMemoryBlobDescriptorCacheProvider(), false, true, false)
 	rootpath, _ := defaultPathMapper.path(repositoriesRootPathSpec{})
 
 	repos := []string{

docs/storage/linkedblobstore.go

@@ -16,11 +16,12 @@ import (
 // that grant access to the global blob store.
 type linkedBlobStore struct {
 	*blobStore
-	blobServer           distribution.BlobServer
-	blobAccessController distribution.BlobDescriptorService
-	repository           distribution.Repository
-	ctx                  context.Context // only to be used where context can't come through method args
-	deleteEnabled        bool
+	blobServer             distribution.BlobServer
+	blobAccessController   distribution.BlobDescriptorService
+	repository             distribution.Repository
+	ctx                    context.Context // only to be used where context can't come through method args
+	deleteEnabled          bool
+	resumableDigestEnabled bool
 
 	// linkPath allows one to control the repository blob link set to which
 	// the blob store dispatches. This is required because manifest and layer
@@ -189,11 +190,12 @@ func (lbs *linkedBlobStore) newBlobUpload(ctx context.Context, uuid, path string
 	}
 
 	bw := &blobWriter{
-		blobStore:          lbs,
-		id:                 uuid,
-		startedAt:          startedAt,
-		digester:           digest.Canonical.New(),
-		bufferedFileWriter: *fw,
+		blobStore:              lbs,
+		id:                     uuid,
+		startedAt:              startedAt,
+		digester:               digest.Canonical.New(),
+		bufferedFileWriter:     *fw,
+		resumableDigestEnabled: lbs.resumableDigestEnabled,
 	}
 
 	return bw, nil

docs/storage/manifeststore_test.go

@@ -29,7 +29,7 @@ type manifestStoreTestEnv struct {
 func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv {
 	ctx := context.Background()
 	driver := inmemory.New()
-	registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), true, true)
+	registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider(), true, true, false)
 
 	repo, err := registry.Repository(ctx, name)
 	if err != nil {
@@ -348,7 +348,7 @@ func TestManifestStorage(t *testing.T) {
 		t.Errorf("Deleted manifest get returned non-nil")
 	}
 
-	r := NewRegistryWithDriver(ctx, env.driver, memory.NewInMemoryBlobDescriptorCacheProvider(), false, true)
+	r := NewRegistryWithDriver(ctx, env.driver, memory.NewInMemoryBlobDescriptorCacheProvider(), false, true, false)
 	repo, err := r.Repository(ctx, env.name)
 	if err != nil {
 		t.Fatalf("unexpected error getting repo: %v", err)

docs/storage/registry.go

@@ -16,6 +16,7 @@ type registry struct {
 	statter                     distribution.BlobStatter // global statter service.
 	blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider
 	deleteEnabled               bool
+	resumableDigestEnabled      bool
 }
 
 // NewRegistryWithDriver creates a new registry instance from the provided
@@ -23,9 +24,9 @@ type registry struct {
 // cheap to allocate. If redirect is true, the backend blob server will
 // attempt to use (StorageDriver).URLFor to serve all blobs.
 //
-// TODO(stevvooe): This function signature is getting out of hand. Move to
+// TODO(stevvooe): This function signature is getting very out of hand. Move to
 // functional options for instance configuration.
-func NewRegistryWithDriver(ctx context.Context, driver storagedriver.StorageDriver, blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider, deleteEnabled bool, redirect bool) distribution.Namespace {
+func NewRegistryWithDriver(ctx context.Context, driver storagedriver.StorageDriver, blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider, deleteEnabled bool, redirect bool, isCache bool) distribution.Namespace {
 	// create global statter, with cache.
 	var statter distribution.BlobDescriptorService = &blobStatter{
 		driver: driver,
@@ -52,6 +53,7 @@ func NewRegistryWithDriver(ctx context.Context, driver storagedriver.StorageDriv
 		},
 		blobDescriptorCacheProvider: blobDescriptorCacheProvider,
 		deleteEnabled:               deleteEnabled,
+		resumableDigestEnabled:      !isCache,
 	}
 }
 

docs/storage/vacuum.go

@@ -0,0 +1,67 @@
+package storage
+
+import (
+	"path"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+// vacuum contains functions for cleaning up repositories and blobs
+// These functions will only reliably work on strongly consistent
+// storage systems.
+// https://en.wikipedia.org/wiki/Consistency_model
+
+// NewVacuum creates a new Vacuum
+func NewVacuum(ctx context.Context, driver driver.StorageDriver) Vacuum {
+	return Vacuum{
+		ctx:    ctx,
+		driver: driver,
+		pm:     defaultPathMapper,
+	}
+}
+
+// Vacuum removes content from the filesystem
+type Vacuum struct {
+	pm     *pathMapper
+	driver driver.StorageDriver
+	ctx    context.Context
+}
+
+// RemoveBlob removes a blob from the filesystem
+func (v Vacuum) RemoveBlob(dgst string) error {
+	d, err := digest.ParseDigest(dgst)
+	if err != nil {
+		return err
+	}
+
+	blobPath, err := v.pm.path(blobDataPathSpec{digest: d})
+	if err != nil {
+		return err
+	}
+	context.GetLogger(v.ctx).Infof("Deleting blob: %s", blobPath)
+	err = v.driver.Delete(v.ctx, blobPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RemoveRepository removes a repository directory from the
+// filesystem
+func (v Vacuum) RemoveRepository(repoName string) error {
+	rootForRepository, err := v.pm.path(repositoriesRootPathSpec{})
+	if err != nil {
+		return err
+	}
+	repoDir := path.Join(rootForRepository, repoName)
+	context.GetLogger(v.ctx).Infof("Deleting repo: %s", repoDir)
+	err = v.driver.Delete(v.ctx, repoDir)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}