From 4688d1adc2ddbef7dac71450e7a0583655073284 Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Thu, 28 Jan 2016 15:48:49 -0800 Subject: [PATCH 1/2] Adds "storageclass" configuration parameter for S3 driver. Defaults to STANDARD, also supports REDUCED_REDUNDANCY. Signed-off-by: Brian Bland --- docs/storage-drivers/s3.md | 27 ++++++++++++++++++++------- registry/storage/driver/s3/s3.go | 26 ++++++++++++++++++++++++-- registry/storage/driver/s3/s3_test.go | 2 ++ 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/docs/storage-drivers/s3.md b/docs/storage-drivers/s3.md index 5b172f9e..a59c9308 100644 --- a/docs/storage-drivers/s3.md +++ b/docs/storage-drivers/s3.md @@ -30,7 +30,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz Your AWS Access Key. - + secretkey @@ -41,7 +41,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz Your AWS Secret Key. - + region @@ -64,7 +64,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz The bucket name in which you want to store the registry's data. - + encrypt @@ -76,7 +76,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz not. A boolean value. The default is false. - + secure @@ -88,7 +88,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz default is true. - + v4auth @@ -101,7 +101,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz false. - + chunksize @@ -113,7 +113,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz should be a number that is larger than 5*1024*1024. - + rootdirectory @@ -124,6 +124,17 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz This is a prefix that will be applied to all S3 keys to allow you to segment data in your bucket if necessary. + + + storageclass + + + no + + + The S3 storage class applied to each registry file. The default value is STANDARD. + + @@ -147,6 +158,8 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz `rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to the empty string (bucket root). +`storageclass`: (optional) The storage class applied to each registry file. Defaults to STANDARD. Valid options are STANDARD and REDUCED_REDUNDANCY. + # CloudFront as Middleware with S3 backend ## Use Case diff --git a/registry/storage/driver/s3/s3.go b/registry/storage/driver/s3/s3.go index f09e5508..83fd74f7 100644 --- a/registry/storage/driver/s3/s3.go +++ b/registry/storage/driver/s3/s3.go @@ -1,7 +1,7 @@ // Package s3 provides a storagedriver.StorageDriver implementation to // store blobs in Amazon S3 cloud storage. // -// This package leverages the AdRoll/goamz client library for interfacing with +// This package leverages the docker/goamz client library for interfacing with // s3. // // Because s3 is a key, value store the Stat call does not support last modification @@ -59,6 +59,7 @@ type DriverParameters struct { V4Auth bool ChunkSize int64 RootDirectory string + StorageClass s3.StorageClass UserAgent string } @@ -79,6 +80,7 @@ type driver struct { ChunkSize int64 Encrypt bool RootDirectory string + StorageClass s3.StorageClass pool sync.Pool // pool []byte buffers used for WriteStream zeros []byte // shared, zero-valued buffer used for WriteStream @@ -183,6 +185,21 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { rootDirectory = "" } + storageClass := s3.StandardStorage + storageClassParam, ok := parameters["storageclass"] + if ok { + storageClassString, ok := storageClassParam.(string) + if !ok { + return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", []s3.StorageClass{s3.StandardStorage, s3.ReducedRedundancy}, storageClassParam) + } + // All valid storage class parameters are UPPERCASE, so be a bit more flexible here + storageClassCasted := s3.StorageClass(strings.ToUpper(storageClassString)) + if storageClassCasted != s3.StandardStorage && storageClassCasted != s3.ReducedRedundancy { + return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", []s3.StorageClass{s3.StandardStorage, s3.ReducedRedundancy}, storageClassParam) + } + storageClass = storageClassCasted + } + userAgent, ok := parameters["useragent"] if !ok { userAgent = "" @@ -198,6 +215,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { v4AuthBool, chunkSize, fmt.Sprint(rootDirectory), + storageClass, fmt.Sprint(userAgent), } @@ -259,6 +277,7 @@ func New(params DriverParameters) (*Driver, error) { ChunkSize: params.ChunkSize, Encrypt: params.Encrypt, RootDirectory: params.RootDirectory, + StorageClass: params.StorageClass, zeros: make([]byte, params.ChunkSize), } @@ -826,7 +845,10 @@ func hasCode(err error, code string) bool { } func (d *driver) getOptions() s3.Options { - return s3.Options{SSE: d.Encrypt} + return s3.Options{ + SSE: d.Encrypt, + StorageClass: d.StorageClass, + } } func getPermissions() s3.ACL { diff --git a/registry/storage/driver/s3/s3_test.go b/registry/storage/driver/s3/s3_test.go index 86f433f3..be099790 100644 --- a/registry/storage/driver/s3/s3_test.go +++ b/registry/storage/driver/s3/s3_test.go @@ -10,6 +10,7 @@ import ( storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/testsuites" "github.com/docker/goamz/aws" + "github.com/docker/goamz/s3" "gopkg.in/check.v1" ) @@ -69,6 +70,7 @@ func init() { v4AuthBool, minChunkSize, rootDirectory, + s3.StandardStorage, "", } From 695bac7b0f4891b613b2e4e2cfd2bdd798f89b7d Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Mon, 1 Feb 2016 15:34:36 -0800 Subject: [PATCH 2/2] Adds test for S3 storage class configuration option Signed-off-by: Brian Bland --- registry/storage/driver/s3/s3_test.go | 74 ++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/registry/storage/driver/s3/s3_test.go b/registry/storage/driver/s3/s3_test.go index be099790..660d5350 100644 --- a/registry/storage/driver/s3/s3_test.go +++ b/registry/storage/driver/s3/s3_test.go @@ -18,7 +18,7 @@ import ( // Hook up gocheck into the "go test" runner. func Test(t *testing.T) { check.TestingT(t) } -var s3DriverConstructor func(rootDirectory string) (*Driver, error) +var s3DriverConstructor func(rootDirectory string, storageClass s3.StorageClass) (*Driver, error) var skipS3 func() string func init() { @@ -35,7 +35,7 @@ func init() { } defer os.Remove(root) - s3DriverConstructor = func(rootDirectory string) (*Driver, error) { + s3DriverConstructor = func(rootDirectory string, storageClass s3.StorageClass) (*Driver, error) { encryptBool := false if encrypt != "" { encryptBool, err = strconv.ParseBool(encrypt) @@ -70,7 +70,7 @@ func init() { v4AuthBool, minChunkSize, rootDirectory, - s3.StandardStorage, + storageClass, "", } @@ -86,7 +86,7 @@ func init() { } testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) { - return s3DriverConstructor(root) + return s3DriverConstructor(root, s3.StandardStorage) }, skipS3) } @@ -101,17 +101,17 @@ func TestEmptyRootList(t *testing.T) { } defer os.Remove(validRoot) - rootedDriver, err := s3DriverConstructor(validRoot) + rootedDriver, err := s3DriverConstructor(validRoot, s3.StandardStorage) if err != nil { t.Fatalf("unexpected error creating rooted driver: %v", err) } - emptyRootDriver, err := s3DriverConstructor("") + emptyRootDriver, err := s3DriverConstructor("", s3.StandardStorage) if err != nil { t.Fatalf("unexpected error creating empty root driver: %v", err) } - slashRootDriver, err := s3DriverConstructor("/") + slashRootDriver, err := s3DriverConstructor("/", s3.StandardStorage) if err != nil { t.Fatalf("unexpected error creating slash root driver: %v", err) } @@ -139,3 +139,63 @@ func TestEmptyRootList(t *testing.T) { } } } + +func TestStorageClass(t *testing.T) { + if skipS3() != "" { + t.Skip(skipS3()) + } + + rootDir, err := ioutil.TempDir("", "driver-") + if err != nil { + t.Fatalf("unexpected error creating temporary directory: %v", err) + } + defer os.Remove(rootDir) + + standardDriver, err := s3DriverConstructor(rootDir, s3.StandardStorage) + if err != nil { + t.Fatalf("unexpected error creating driver with standard storage: %v", err) + } + + rrDriver, err := s3DriverConstructor(rootDir, s3.ReducedRedundancy) + if err != nil { + t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err) + } + + standardFilename := "/test-standard" + rrFilename := "/test-rr" + contents := []byte("contents") + ctx := context.Background() + + err = standardDriver.PutContent(ctx, standardFilename, contents) + if err != nil { + t.Fatalf("unexpected error creating content: %v", err) + } + defer standardDriver.Delete(ctx, standardFilename) + + err = rrDriver.PutContent(ctx, rrFilename, contents) + if err != nil { + t.Fatalf("unexpected error creating content: %v", err) + } + defer rrDriver.Delete(ctx, rrFilename) + + standardDriverUnwrapped := standardDriver.Base.StorageDriver.(*driver) + resp, err := standardDriverUnwrapped.Bucket.GetResponse(standardDriverUnwrapped.s3Path(standardFilename)) + if err != nil { + t.Fatalf("unexpected error retrieving standard storage file: %v", err) + } + defer resp.Body.Close() + // Amazon only populates this header value for non-standard storage classes + if storageClass := resp.Header.Get("x-amz-storage-class"); storageClass != "" { + t.Fatalf("unexpected storage class for standard file: %v", storageClass) + } + + rrDriverUnwrapped := rrDriver.Base.StorageDriver.(*driver) + resp, err = rrDriverUnwrapped.Bucket.GetResponse(rrDriverUnwrapped.s3Path(rrFilename)) + if err != nil { + t.Fatalf("unexpected error retrieving reduced-redundancy storage file: %v", err) + } + defer resp.Body.Close() + if storageClass := resp.Header.Get("x-amz-storage-class"); storageClass != string(s3.ReducedRedundancy) { + t.Fatalf("unexpected storage class for standard file: %v", storageClass) + } +}