diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 7d6d0700..8d9b39b4 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -7,15 +7,15 @@ "Deps": [ { "ImportPath": "github.com/AdRoll/goamz/aws", - "Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989" + "Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1" }, { "ImportPath": "github.com/AdRoll/goamz/cloudfront", - "Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989" + "Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1" }, { "ImportPath": "github.com/AdRoll/goamz/s3", - "Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989" + "Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1" }, { "ImportPath": "github.com/Azure/azure-sdk-for-go/storage", diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go index 87c2d6da..9939f92c 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go @@ -51,7 +51,7 @@ type ServiceInfo struct { // See http://goo.gl/d8BP1 for more details. type Region struct { Name string // the canonical name of this region. - EC2Endpoint string + EC2Endpoint ServiceInfo S3Endpoint string S3BucketEndpoint string // Not needed by AWS S3. Use ${bucket} for bucket name. S3LocationConstraint bool // true if this region requires a LocationConstraint declaration. diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go index fdc2626b..7755bc7f 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go @@ -2,7 +2,7 @@ package aws var USGovWest = Region{ "us-gov-west-1", - "https://ec2.us-gov-west-1.amazonaws.com", + ServiceInfo{"https://ec2.us-gov-west-1.amazonaws.com", V2Signature}, "https://s3-fips-us-gov-west-1.amazonaws.com", "", true, @@ -26,8 +26,8 @@ var USGovWest = Region{ var USEast = Region{ "us-east-1", - "https://ec2.us-east-1.amazonaws.com", - "https://s3.amazonaws.com", + ServiceInfo{"https://ec2.us-east-1.amazonaws.com", V2Signature}, + "https://s3-external-1.amazonaws.com", "", false, false, @@ -50,7 +50,7 @@ var USEast = Region{ var USWest = Region{ "us-west-1", - "https://ec2.us-west-1.amazonaws.com", + ServiceInfo{"https://ec2.us-west-1.amazonaws.com", V2Signature}, "https://s3-us-west-1.amazonaws.com", "", true, @@ -74,7 +74,7 @@ var USWest = Region{ var USWest2 = Region{ "us-west-2", - "https://ec2.us-west-2.amazonaws.com", + ServiceInfo{"https://ec2.us-west-2.amazonaws.com", V2Signature}, "https://s3-us-west-2.amazonaws.com", "", true, @@ -98,7 +98,7 @@ var USWest2 = Region{ var EUWest = Region{ "eu-west-1", - "https://ec2.eu-west-1.amazonaws.com", + ServiceInfo{"https://ec2.eu-west-1.amazonaws.com", V2Signature}, "https://s3-eu-west-1.amazonaws.com", "", true, @@ -122,7 +122,7 @@ var EUWest = Region{ var EUCentral = Region{ "eu-central-1", - "https://ec2.eu-central-1.amazonaws.com", + ServiceInfo{"https://ec2.eu-central-1.amazonaws.com", V4Signature}, "https://s3-eu-central-1.amazonaws.com", "", true, @@ -146,7 +146,7 @@ var EUCentral = Region{ var APSoutheast = Region{ "ap-southeast-1", - "https://ec2.ap-southeast-1.amazonaws.com", + ServiceInfo{"https://ec2.ap-southeast-1.amazonaws.com", V2Signature}, "https://s3-ap-southeast-1.amazonaws.com", "", true, @@ -170,7 +170,7 @@ var APSoutheast = Region{ var APSoutheast2 = Region{ "ap-southeast-2", - "https://ec2.ap-southeast-2.amazonaws.com", + ServiceInfo{"https://ec2.ap-southeast-2.amazonaws.com", V2Signature}, "https://s3-ap-southeast-2.amazonaws.com", "", true, @@ -194,7 +194,7 @@ var APSoutheast2 = Region{ var APNortheast = Region{ "ap-northeast-1", - "https://ec2.ap-northeast-1.amazonaws.com", + ServiceInfo{"https://ec2.ap-northeast-1.amazonaws.com", V2Signature}, "https://s3-ap-northeast-1.amazonaws.com", "", true, @@ -218,7 +218,7 @@ var APNortheast = Region{ var SAEast = Region{ "sa-east-1", - "https://ec2.sa-east-1.amazonaws.com", + ServiceInfo{"https://ec2.sa-east-1.amazonaws.com", V2Signature}, "https://s3-sa-east-1.amazonaws.com", "", true, @@ -242,7 +242,7 @@ var SAEast = Region{ var CNNorth1 = Region{ "cn-north-1", - "https://ec2.cn-north-1.amazonaws.com.cn", + ServiceInfo{"https://ec2.cn-north-1.amazonaws.com.cn", V2Signature}, "https://s3.cn-north-1.amazonaws.com.cn", "", true, diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go index 5875beee..ea296013 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go @@ -15,6 +15,28 @@ import ( "time" ) +// AWS specifies that the parameters in a signed request must +// be provided in the natural order of the keys. This is distinct +// from the natural order of the encoded value of key=value. +// Percent and gocheck.Equals affect the sorting order. +func EncodeSorted(values url.Values) string { + // preallocate the arrays for perfomance + keys := make([]string, 0, len(values)) + sarray := make([]string, 0, len(values)) + for k, _ := range values { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + for _, v := range values[k] { + sarray = append(sarray, Encode(k)+"="+Encode(v)) + } + } + + return strings.Join(sarray, "&") +} + type V2Signer struct { auth Auth service ServiceInfo @@ -38,7 +60,6 @@ func (s *V2Signer) Sign(method, path string, params map[string]string) { if s.auth.Token() != "" { params["SecurityToken"] = s.auth.Token() } - // AWS specifies that the parameters in a signed request must // be provided in the natural order of the keys. This is distinct // from the natural order of the encoded value of key=value. @@ -61,6 +82,28 @@ func (s *V2Signer) Sign(method, path string, params map[string]string) { params["Signature"] = string(signature) } +func (s *V2Signer) SignRequest(req *http.Request) error { + req.ParseForm() + req.Form.Set("AWSAccessKeyId", s.auth.AccessKey) + req.Form.Set("SignatureVersion", "2") + req.Form.Set("SignatureMethod", "HmacSHA256") + if s.auth.Token() != "" { + req.Form.Set("SecurityToken", s.auth.Token()) + } + + payload := req.Method + "\n" + req.URL.Host + "\n" + req.URL.Path + "\n" + EncodeSorted(req.Form) + hash := hmac.New(sha256.New, []byte(s.auth.SecretKey)) + hash.Write([]byte(payload)) + signature := make([]byte, b64.EncodedLen(hash.Size())) + b64.Encode(signature, hash.Sum(nil)) + + req.Form.Set("Signature", string(signature)) + + req.URL.RawQuery = req.Form.Encode() + + return nil +} + // Common date formats for signing requests const ( ISO8601BasicFormat = "20060102T150405Z" @@ -174,6 +217,11 @@ func (s *V4Signer) Sign(req *http.Request) { return } +func (s *V4Signer) SignRequest(req *http.Request) error { + s.Sign(req) + return nil +} + /* requestTime method will parse the time from the request "x-amz-date" or "date" headers. If the "x-amz-date" header is present, that will take priority over the "date" header. diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi.go index ff454265..d905f565 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi.go @@ -8,6 +8,7 @@ import ( "encoding/xml" "errors" "io" + "net/http" "net/url" "sort" "strconv" @@ -428,6 +429,11 @@ func (m *Multi) Complete(parts []Part) error { payload: bytes.NewReader(data), } var resp completeUploadResp + if m.Bucket.Region.Name == "generic" { + headers := make(http.Header) + headers.Add("Content-Length", strconv.FormatInt(int64(len(data)), 10)) + req.headers = headers + } err := m.Bucket.S3.query(req, &resp) if shouldRetry(err) && attempt.HasNext() { continue diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go index 8429d336..b76f70eb 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go @@ -2,11 +2,12 @@ package s3_test import ( "encoding/xml" - "github.com/AdRoll/goamz/s3" - "gopkg.in/check.v1" "io" "io/ioutil" "strings" + + "github.com/AdRoll/goamz/s3" + "gopkg.in/check.v1" ) func (s *S) TestInitMulti(c *check.C) { @@ -438,3 +439,42 @@ func (s *S) TestListMulti(c *check.C) { c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"}) c.Assert(req.Form["max-uploads"], check.DeepEquals, []string{"1000"}) } + +func (s *S) TestMultiCompleteSupportRadosGW(c *check.C) { + testServer.Response(200, nil, InitMultiResultDump) + testServer.Response(200, nil, MultiCompleteDump) + s.s3.Region.Name = "generic" + b := s.s3.Bucket("sample") + + multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) + c.Assert(err, check.IsNil) + + err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}}) + c.Assert(err, check.IsNil) + + testServer.WaitRequest() + req := testServer.WaitRequest() + c.Assert(req.Method, check.Equals, "POST") + c.Assert(req.URL.Path, check.Equals, "/sample/multi") + c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") + c.Assert(req.Header["Content-Length"], check.NotNil) + + var payload struct { + XMLName xml.Name + Part []struct { + PartNumber int + ETag string + } + } + + dec := xml.NewDecoder(req.Body) + err = dec.Decode(&payload) + c.Assert(err, check.IsNil) + + c.Assert(payload.XMLName.Local, check.Equals, "CompleteMultipartUpload") + c.Assert(len(payload.Part), check.Equals, 2) + c.Assert(payload.Part[0].PartNumber, check.Equals, 1) + c.Assert(payload.Part[0].ETag, check.Equals, `"ETag1"`) + c.Assert(payload.Part[1].PartNumber, check.Equals, 2) + c.Assert(payload.Part[1].ETag, check.Equals, `"ETag2"`) +} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go index dd313025..c8d4a570 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go @@ -1074,11 +1074,14 @@ func (s3 *S3) prepare(req *request) error { return err } - signpathPatiallyEscaped := partiallyEscapedPath(req.path) + signpathPartiallyEscaped := partiallyEscapedPath(req.path) + if strings.IndexAny(s3.Region.S3BucketEndpoint, "${bucket}") >= 0 { + signpathPartiallyEscaped = "/" + req.bucket + signpathPartiallyEscaped + } req.headers["Host"] = []string{u.Host} req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)} - sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers) + sign(s3.Auth, req.method, signpathPartiallyEscaped, req.params, req.headers) } else { hreq, err := s3.setupHttpRequest(req) if err != nil { @@ -1111,7 +1114,9 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) { if err != nil { return nil, err } - u.Opaque = fmt.Sprintf("//%s%s", u.Host, partiallyEscapedPath(u.Path)) + if s3.Region.Name != "generic" { + u.Opaque = fmt.Sprintf("//%s%s", u.Host, partiallyEscapedPath(u.Path)) + } hreq := http.Request{ URL: u, diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go index 161bb3af..7a17474c 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go @@ -500,3 +500,14 @@ func (s *S) TestLocation(c *check.C) { c.Assert(err, check.IsNil) c.Assert(resultUsWest1, check.Equals, expectedUsWest1) } + +func (s *S) TestSupportRadosGW(c *check.C) { + testServer.Response(200, nil, "content") + s.s3.Region.Name = "generic" + b := s.s3.Bucket("bucket") + _, err := b.Get("rgw") + + req := testServer.WaitRequest() + c.Assert(err, check.IsNil) + c.Assert(req.RequestURI, check.Equals, "/bucket/rgw") +} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go index 0dd63af3..2e4e3f24 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go @@ -316,7 +316,8 @@ func (nullResource) get(a *action) interface{} { return notAllowed() } func (nullResource) post(a *action) interface{} { return notAllowed() } func (nullResource) delete(a *action) interface{} { return notAllowed() } -const timeFormat = "2006-01-02T15:04:05.000Z07:00" +const timeFormat = "2006-01-02T15:04:05.000Z" +const lastModifiedTimeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" type bucketResource struct { name string @@ -418,7 +419,7 @@ func (s orderedObjects) Less(i, j int) bool { func (obj *object) s3Key() s3.Key { return s3.Key{ Key: obj.name, - LastModified: obj.mtime.Format(timeFormat), + LastModified: obj.mtime.UTC().Format(timeFormat), Size: int64(len(obj.data)), ETag: fmt.Sprintf(`"%x"`, obj.checksum), // TODO StorageClass @@ -647,8 +648,8 @@ func (objr objectResource) get(a *action) interface{} { // TODO Connection: close ?? // TODO x-amz-request-id h.Set("Content-Length", fmt.Sprint(len(data))) - h.Set("ETag", hex.EncodeToString(obj.checksum)) - h.Set("Last-Modified", obj.mtime.Format(time.RFC1123)) + h.Set("ETag", "\""+hex.EncodeToString(obj.checksum)+"\"") + h.Set("Last-Modified", obj.mtime.Format(lastModifiedTimeFormat)) if status != http.StatusOK { a.w.WriteHeader(status) @@ -750,8 +751,45 @@ func (objr objectResource) put(a *action) interface{} { obj.meta[key] = values } } - obj.data = data - obj.checksum = gotHash + + if copySource := a.req.Header.Get("X-Amz-Copy-Source"); copySource != "" { + idx := strings.IndexByte(copySource, '/') + + if idx == -1 { + fatalf(400, "InvalidRequest", "Wrongly formatted X-Amz-Copy-Source") + } + + sourceBucketName := copySource[0:idx] + sourceKey := copySource[1+idx:] + + sourceBucket := a.srv.buckets[sourceBucketName] + + if sourceBucket == nil { + fatalf(404, "NoSuchBucket", "The specified source bucket does not exist") + } + + sourceObject := sourceBucket.objects[sourceKey] + + if sourceObject == nil { + fatalf(404, "NoSuchKey", "The specified source key does not exist") + } + + obj.data = make([]byte, len(sourceObject.data)) + copy(obj.data, sourceObject.data) + + obj.checksum = make([]byte, len(sourceObject.checksum)) + copy(obj.checksum, sourceObject.checksum) + + obj.meta = make(http.Header, len(sourceObject.meta)) + + for k, v := range sourceObject.meta { + obj.meta[k] = make([]string, len(v)) + copy(obj.meta[k], v) + } + } else { + obj.data = data + obj.checksum = gotHash + } obj.mtime = time.Now() objr.bucket.objects[objr.name] = obj } else {