Manifest and layer soft deletion.

Implement the delete API by implementing soft delete for layers
and blobs by removing link files and updating the blob descriptor
cache.  Deletion is configurable - if it is disabled API calls
will return an unsupported error.

We invalidate the blob descriptor cache by changing the linkedBlobStore's
blobStatter to a blobDescriptorService and naming it blobAccessController.

Delete() is added throughout the relevant API to support this functionality.

Signed-off-by: Richard Scothern <richard.scothern@gmail.com>

docs/storage/cache/cachedblobdescriptorstore.go

@@ -26,13 +26,13 @@ type MetricsTracker interface {
 
 type cachedBlobStatter struct {
 	cache   distribution.BlobDescriptorService
-	backend distribution.BlobStatter
+	backend distribution.BlobDescriptorService
 	tracker MetricsTracker
 }
 
 // NewCachedBlobStatter creates a new statter which prefers a cache and
 // falls back to a backend.
-func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend distribution.BlobStatter) distribution.BlobStatter {
+func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService) distribution.BlobDescriptorService {
 	return &cachedBlobStatter{
 		cache:   cache,
 		backend: backend,
@@ -41,7 +41,7 @@ func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend dist
 
 // NewCachedBlobStatterWithMetrics creates a new statter which prefers a cache and
 // falls back to a backend. Hits and misses will send to the tracker.
-func NewCachedBlobStatterWithMetrics(cache distribution.BlobDescriptorService, backend distribution.BlobStatter, tracker MetricsTracker) distribution.BlobStatter {
+func NewCachedBlobStatterWithMetrics(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService, tracker MetricsTracker) distribution.BlobStatter {
 	return &cachedBlobStatter{
 		cache:   cache,
 		backend: backend,
@@ -77,4 +77,25 @@ fallback:
 	}
 
 	return desc, err
+
+}
+
+func (cbds *cachedBlobStatter) Clear(ctx context.Context, dgst digest.Digest) error {
+	err := cbds.cache.Clear(ctx, dgst)
+	if err != nil {
+		return err
+	}
+
+	err = cbds.backend.Clear(ctx, dgst)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cbds *cachedBlobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil {
+		context.GetLogger(ctx).Errorf("error adding descriptor %v to cache: %v", desc.Digest, err)
+	}
+	return nil
 }

docs/storage/cache/memory/memory.go

@@ -44,6 +44,10 @@ func (imbdcp *inMemoryBlobDescriptorCacheProvider) Stat(ctx context.Context, dgs
 	return imbdcp.global.Stat(ctx, dgst)
 }
 
+func (imbdcp *inMemoryBlobDescriptorCacheProvider) Clear(ctx context.Context, dgst digest.Digest) error {
+	return imbdcp.global.Clear(ctx, dgst)
+}
+
 func (imbdcp *inMemoryBlobDescriptorCacheProvider) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
 	_, err := imbdcp.Stat(ctx, dgst)
 	if err == distribution.ErrBlobUnknown {
@@ -80,6 +84,14 @@ func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) Stat(ctx context.Co
 	return rsimbdcp.repository.Stat(ctx, dgst)
 }
 
+func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) Clear(ctx context.Context, dgst digest.Digest) error {
+	if rsimbdcp.repository == nil {
+		return distribution.ErrBlobUnknown
+	}
+
+	return rsimbdcp.repository.Clear(ctx, dgst)
+}
+
 func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
 	if rsimbdcp.repository == nil {
 		// allocate map since we are setting it now.
@@ -133,6 +145,14 @@ func (mbdc *mapBlobDescriptorCache) Stat(ctx context.Context, dgst digest.Digest
 	return desc, nil
 }
 
+func (mbdc *mapBlobDescriptorCache) Clear(ctx context.Context, dgst digest.Digest) error {
+	mbdc.mu.Lock()
+	defer mbdc.mu.Unlock()
+
+	delete(mbdc.descriptors, dgst)
+	return nil
+}
+
 func (mbdc *mapBlobDescriptorCache) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
 	if err := dgst.Validate(); err != nil {
 		return err

docs/storage/cache/redis/redis.go

@@ -12,7 +12,7 @@ import (
 )
 
 // redisBlobStatService provides an implementation of
-// BlobDescriptorCacheProvider based on redis. Blob descritors are stored in
+// BlobDescriptorCacheProvider based on redis. Blob descriptors are stored in
 // two parts. The first provide fast access to repository membership through a
 // redis set for each repo. The second is a redis hash keyed by the digest of
 // the layer, providing path, length and mediatype information. There is also
@@ -63,6 +63,27 @@ func (rbds *redisBlobDescriptorService) Stat(ctx context.Context, dgst digest.Di
 	return rbds.stat(ctx, conn, dgst)
 }
 
+func (rbds *redisBlobDescriptorService) Clear(ctx context.Context, dgst digest.Digest) error {
+	if err := dgst.Validate(); err != nil {
+		return err
+	}
+
+	conn := rbds.pool.Get()
+	defer conn.Close()
+
+	// Not atomic in redis <= 2.3
+	reply, err := conn.Do("HDEL", rbds.blobDescriptorHashKey(dgst), "digest", "length", "mediatype")
+	if err != nil {
+		return err
+	}
+
+	if reply == 0 {
+		return distribution.ErrBlobUnknown
+	}
+
+	return nil
+}
+
 // stat provides an internal stat call that takes a connection parameter. This
 // allows some internal management of the connection scope.
 func (rbds *redisBlobDescriptorService) stat(ctx context.Context, conn redis.Conn, dgst digest.Digest) (distribution.Descriptor, error) {
@@ -170,6 +191,28 @@ func (rsrbds *repositoryScopedRedisBlobDescriptorService) Stat(ctx context.Conte
 	return upstream, nil
 }
 
+// Clear removes the descriptor from the cache and forwards to the upstream descriptor store
+func (rsrbds *repositoryScopedRedisBlobDescriptorService) Clear(ctx context.Context, dgst digest.Digest) error {
+	if err := dgst.Validate(); err != nil {
+		return err
+	}
+
+	conn := rsrbds.upstream.pool.Get()
+	defer conn.Close()
+
+	// Check membership to repository first
+	member, err := redis.Bool(conn.Do("SISMEMBER", rsrbds.repositoryBlobSetKey(rsrbds.repo), dgst))
+	if err != nil {
+		return err
+	}
+
+	if !member {
+		return distribution.ErrBlobUnknown
+	}
+
+	return rsrbds.upstream.Clear(ctx, dgst)
+}
+
 func (rsrbds *repositoryScopedRedisBlobDescriptorService) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
 	if err := dgst.Validate(); err != nil {
 		return err

docs/storage/cache/suite.go

@@ -139,3 +139,40 @@ func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provi
 		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
 	}
 }
+
+func checkBlobDescriptorClear(t *testing.T, ctx context.Context, provider BlobDescriptorCacheProvider) {
+	localDigest := digest.Digest("sha384:abc")
+	expected := distribution.Descriptor{
+		Digest:    "sha256:abc",
+		Size:      10,
+		MediaType: "application/octet-stream"}
+
+	cache, err := provider.RepositoryScoped("foo/bar")
+	if err != nil {
+		t.Fatalf("unexpected error getting scoped cache: %v", err)
+	}
+
+	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
+		t.Fatalf("error setting descriptor: %v", err)
+	}
+
+	desc, err := cache.Stat(ctx, localDigest)
+	if err != nil {
+		t.Fatalf("unexpected error statting fake2:abc: %v", err)
+	}
+
+	if expected != desc {
+		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
+	}
+
+	err = cache.Clear(ctx, localDigest)
+	if err != nil {
+		t.Fatalf("unexpected error deleting descriptor")
+	}
+
+	nonExistantDigest := digest.Digest("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+	err = cache.Clear(ctx, nonExistantDigest)
+	if err == nil {
+		t.Fatalf("expected error deleting unknown descriptor")
+	}
+}