From 2be1b4ef4f3b489b3dcaeb21ed20ec6a641b4070 Mon Sep 17 00:00:00 2001 From: Keerthan Mala Date: Sat, 5 Mar 2016 11:46:44 -0700 Subject: [PATCH] Added support to specifiy custom endpoint Signed-off-by: Keerthan Reddy Mala --- docs/configuration.md | 4 +- docs/storage-drivers/s3.md | 15 +++++- registry/storage/driver/s3-aws/s3.go | 64 +++++++++++++++-------- registry/storage/driver/s3-aws/s3_test.go | 2 + 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ef01d2f6..67b608b8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -90,6 +90,7 @@ information about each option that appears later in this page. accesskey: awsaccesskey secretkey: awssecretkey region: us-west-1 + regionendpoint: http://myobjects.local bucket: bucketname encrypt: true secure: true @@ -358,6 +359,7 @@ Permitted values are `error`, `warn`, `info` and `debug`. The default is accesskey: awsaccesskey secretkey: awssecretkey region: us-west-1 + regionendpoint: http://myobjects.local bucket: bucketname encrypt: true secure: true @@ -435,7 +437,7 @@ You must configure one backend; if you configure more, the registry returns an e s3 - Uses Amazon's Simple Storage Service (S3). + Uses Amazon's Simple Storage Service (S3) and compatible Storage Services. See the driver's reference documentation. diff --git a/docs/storage-drivers/s3.md b/docs/storage-drivers/s3.md index a59c9308..7ee70657 100644 --- a/docs/storage-drivers/s3.md +++ b/docs/storage-drivers/s3.md @@ -9,7 +9,7 @@ keywords = ["registry, service, driver, images, storage, S3"] # S3 storage driver -An implementation of the `storagedriver.StorageDriver` interface which uses Amazon S3 for object storage. +An implementation of the `storagedriver.StorageDriver` interface which uses Amazon S3 or S3 compatible services for object storage. ## Parameters @@ -52,6 +52,17 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz The AWS region in which your bucket exists. For the moment, the Go AWS library in use does not use the newer DNS based bucket routing. + + + + regionendpoint + + + no + + + Endpoint for S3 compatible storage services (Minio, etc) + @@ -146,6 +157,8 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz `region`: The name of the aws region in which you would like to store objects (for example `us-east-1`). For a list of regions, you can look at http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html +`regionendpoint`: (optional) Endpoint URL for S3 compatible APIs. This should not be provided when using Amazon S3. + `bucket`: The name of your S3 bucket where you wish to store objects. The bucket must exist prior to the driver initialization. `encrypt`: (optional) Whether you would like your data encrypted on the server side (defaults to false if not specified). diff --git a/registry/storage/driver/s3-aws/s3.go b/registry/storage/driver/s3-aws/s3.go index 0e113680..5496311d 100644 --- a/registry/storage/driver/s3-aws/s3.go +++ b/registry/storage/driver/s3-aws/s3.go @@ -56,16 +56,17 @@ var validRegions = map[string]struct{}{} //DriverParameters A struct that encapsulates all of the driver parameters after all values have been set type DriverParameters struct { - AccessKey string - SecretKey string - Bucket string - Region string - Encrypt bool - Secure bool - ChunkSize int64 - RootDirectory string - StorageClass string - UserAgent string + AccessKey string + SecretKey string + Bucket string + Region string + RegionEndpoint string + Encrypt bool + Secure bool + ChunkSize int64 + RootDirectory string + StorageClass string + UserAgent string } func init() { @@ -153,6 +154,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { return nil, fmt.Errorf("No bucket parameter provided") } + regionEndpoint := parameters["regionendpoint"] + if regionEndpoint == nil { + regionEndpoint = "" + } + encryptBool := false encrypt := parameters["encrypt"] switch encrypt := encrypt.(type) { @@ -240,6 +246,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { fmt.Sprint(secretKey), fmt.Sprint(bucket), region, + fmt.Sprint(regionEndpoint), encryptBool, secureBool, chunkSize, @@ -255,22 +262,37 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { // bucketName func New(params DriverParameters) (*Driver, error) { awsConfig := aws.NewConfig() - creds := credentials.NewChainCredentials([]credentials.Provider{ - &credentials.StaticProvider{ - Value: credentials.Value{ - AccessKeyID: params.AccessKey, - SecretAccessKey: params.SecretKey, + var creds *credentials.Credentials + if params.RegionEndpoint == "" { + creds = credentials.NewChainCredentials([]credentials.Provider{ + &credentials.StaticProvider{ + Value: credentials.Value{ + AccessKeyID: params.AccessKey, + SecretAccessKey: params.SecretKey, + }, }, - }, - &credentials.EnvProvider{}, - &credentials.SharedCredentialsProvider{}, - &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())}, - }) + &credentials.EnvProvider{}, + &credentials.SharedCredentialsProvider{}, + &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())}, + }) + + } else { + creds = credentials.NewChainCredentials([]credentials.Provider{ + &credentials.StaticProvider{ + Value: credentials.Value{ + AccessKeyID: params.AccessKey, + SecretAccessKey: params.SecretKey, + }, + }, + &credentials.EnvProvider{}, + }) + awsConfig.WithS3ForcePathStyle(true) + awsConfig.WithEndpoint(params.RegionEndpoint) + } awsConfig.WithCredentials(creds) awsConfig.WithRegion(params.Region) awsConfig.WithDisableSSL(!params.Secure) - // awsConfig.WithMaxRetries(10) if params.UserAgent != "" { awsConfig.WithHTTPClient(&http.Client{ diff --git a/registry/storage/driver/s3-aws/s3_test.go b/registry/storage/driver/s3-aws/s3_test.go index 1137b3e2..f12297bf 100644 --- a/registry/storage/driver/s3-aws/s3_test.go +++ b/registry/storage/driver/s3-aws/s3_test.go @@ -30,6 +30,7 @@ func init() { secure := os.Getenv("S3_SECURE") region := os.Getenv("AWS_REGION") root, err := ioutil.TempDir("", "driver-") + regionEndpoint := os.Getenv("REGION_ENDPOINT") if err != nil { panic(err) } @@ -57,6 +58,7 @@ func init() { secretKey, bucket, region, + regionEndpoint, encryptBool, secureBool, minChunkSize,