Merge pull request #18 from AndreyKostov/ng-add-s3driver-features

Add features to the s3 driver

storagedriver/s3/README.md

@@ -15,8 +15,10 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
 
 `bucket`: The name of your s3 bucket where you wish to store objects (needs to already be created prior to driver initialization).
 
-`encrypt`: (optional) Whether you would like your data encrypted on the server side (defaults to true if not specified).
+`encrypt`: (optional) Whether you would like your data encrypted on the server side (defaults to false if not specified).
 
 `secure`: (optional) Whether you would like to transfer data over ssl or not. Defaults to true (meaning transfering over ssl) if not specified. Note that while setting this to false will improve performance, it is not recommended due to security concerns.
 
+`v4auth`: (optional) Whether you would like to use aws signature version 4 with your requests. This defaults to true if not specified (note that the eu-central-1 region does not work with version 2 signatures, so the driver will error out if initialized with this region and v4auth set to false)
+
 `rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to the empty string (bucket root).

storagedriver/s3/s3.go

@@ -39,6 +39,18 @@ const chunkSize = 5 * 1024 * 1024
 // listMax is the largest amount of objects you can request from S3 in a list call
 const listMax = 1000
 
+//DriverParameters A struct that encapsulates all of the driver parameters after all values have been set
+type DriverParameters struct {
+	AccessKey     string
+	SecretKey     string
+	Bucket        string
+	Region        aws.Region
+	Encrypt       bool
+	Secure        bool
+	V4Auth        bool
+	RootDirectory string
+}
+
 func init() {
 	factory.Register(driverName, &s3DriverFactory{})
 }
@@ -87,7 +99,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
 		return nil, fmt.Errorf("No bucket parameter provided")
 	}
 
-	encryptBool := true
+	encryptBool := false
 	encrypt, ok := parameters["encrypt"]
 	if ok {
 		encryptBool, ok = encrypt.(bool)
@@ -96,24 +108,65 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
 		}
 	}
 
+	secureBool := true
+	secure, ok := parameters["secure"]
+	if ok {
+		secureBool, ok = secure.(bool)
+		if !ok {
+			return nil, fmt.Errorf("The secure parameter should be a boolean")
+		}
+	}
+
+	v4AuthBool := true
+	v4Auth, ok := parameters["v4auth"]
+	if ok {
+		v4AuthBool, ok = v4Auth.(bool)
+		if !ok {
+			return nil, fmt.Errorf("The v4auth parameter should be a boolean")
+		}
+	}
+
 	rootDirectory, ok := parameters["rootdirectory"]
 	if !ok {
 		rootDirectory = ""
 	}
 
-	return New(fmt.Sprint(accessKey), fmt.Sprint(secretKey), fmt.Sprint(bucket), fmt.Sprint(rootDirectory), region, encryptBool)
+	params := DriverParameters{
+		fmt.Sprint(accessKey),
+		fmt.Sprint(secretKey),
+		fmt.Sprint(bucket),
+		region,
+		encryptBool,
+		secureBool,
+		v4AuthBool,
+		fmt.Sprint(rootDirectory),
+	}
+
+	return New(params)
 }
 
 // New constructs a new Driver with the given AWS credentials, region, encryption flag, and
 // bucketName
-func New(accessKey, secretKey, bucketName, rootDirectory string, region aws.Region, encrypt bool) (*Driver, error) {
-	auth, err := aws.GetAuth(accessKey, secretKey, "", time.Time{})
+func New(params DriverParameters) (*Driver, error) {
+	auth, err := aws.GetAuth(params.AccessKey, params.SecretKey, "", time.Time{})
 	if err != nil {
 		return nil, err
 	}
 
-	s3obj := s3.New(auth, region)
-	bucket := s3obj.Bucket(bucketName)
+	if !params.Secure {
+		params.Region.S3Endpoint = strings.Replace(params.Region.S3Endpoint, "https", "http", 1)
+	}
+
+	s3obj := s3.New(auth, params.Region)
+	bucket := s3obj.Bucket(params.Bucket)
+
+	if params.V4Auth {
+		s3obj.Signature = aws.V4Signature
+	} else {
+		if params.Region.Name == "eu-central-1" {
+			return nil, fmt.Errorf("The eu-central-1 region only works with v4 authentication")
+		}
+	}
 
 	if _, err := bucket.List("", "", "", 1); err != nil {
 		return nil, err
@@ -134,7 +187,7 @@ func New(accessKey, secretKey, bucketName, rootDirectory string, region aws.Regi
 	// 	}
 	// }
 
-	return &Driver{s3obj, bucket, encrypt, rootDirectory}, nil
+	return &Driver{s3obj, bucket, params.Encrypt, params.RootDirectory}, nil
 }
 
 // Implement the storagedriver.StorageDriver interface
@@ -413,7 +466,7 @@ func (d *Driver) WriteStream(path string, offset int64, reader io.Reader) (total
 			} else {
 				// offset > currentLength >= chunkSize
 				_, part, err = multi.PutPartCopy(partNumber,
-					s3.CopyOptions{CopySourceOptions: "bytes=0-" + strconv.FormatInt(currentLength-1, 10)},
+					s3.CopyOptions{},
 					d.Bucket.Name+"/"+d.s3Path(path))
 				if err != nil {
 					return 0, err

storagedriver/s3/s3_test.go

@@ -21,18 +21,52 @@ func init() {
 	secretKey := os.Getenv("AWS_SECRET_KEY")
 	bucket := os.Getenv("S3_BUCKET")
 	encrypt := os.Getenv("S3_ENCRYPT")
+	secure := os.Getenv("S3_SECURE")
+	v4auth := os.Getenv("S3_USE_V4_AUTH")
 	region := os.Getenv("AWS_REGION")
 	root, err := ioutil.TempDir("", "driver-")
 	if err != nil {
 		panic(err)
 	}
+	defer os.Remove(root)
 
 	s3DriverConstructor := func(region aws.Region) (storagedriver.StorageDriver, error) {
-		shouldEncrypt, err := strconv.ParseBool(encrypt)
-		if err != nil {
-			return nil, err
+		encryptBool := false
+		if encrypt != "" {
+			encryptBool, err = strconv.ParseBool(encrypt)
+			if err != nil {
+				return nil, err
+			}
 		}
-		return New(accessKey, secretKey, bucket, root, region, shouldEncrypt)
+
+		secureBool := true
+		if secure != "" {
+			secureBool, err = strconv.ParseBool(secure)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		v4AuthBool := true
+		if v4auth != "" {
+			v4AuthBool, err = strconv.ParseBool(v4auth)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		parameters := DriverParameters{
+			accessKey,
+			secretKey,
+			bucket,
+			region,
+			encryptBool,
+			secureBool,
+			v4AuthBool,
+			root,
+		}
+
+		return New(parameters)
 	}
 
 	// Skip S3 storage driver tests if environment variable parameters are not provided