From 2dc1af12a1df1219fbca30784693cd7db7b5482c Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Wed, 20 Jan 2016 16:40:58 -0800 Subject: [PATCH] Adds custom registry User-Agent header to s3 HTTP requests Uses docker/goamz instead of AdRoll/goamz Adds a registry UA string param to the storage parameters when constructing the storage driver for the registry App. This could be used by other storage drivers as well Signed-off-by: Brian Bland --- Godeps/Godeps.json | 24 +- .../AdRoll/goamz/aws/attempt_test.go | 57 -- .../github.com/AdRoll/goamz/aws/aws_test.go | 140 ---- .../AdRoll/goamz/aws/export_test.go | 29 - .../github.com/AdRoll/goamz/aws/retry_test.go | 303 --------- .../github.com/AdRoll/goamz/aws/sign_test.go | 569 ----------------- .../goamz/cloudfront/cloudfront_test.go | 52 -- .../AdRoll/goamz/cloudfront/testdata/key.pub | 6 - .../github.com/AdRoll/goamz/s3/export_test.go | 27 - .../AdRoll/goamz/s3/lifecycle_test.go | 205 ------ .../github.com/AdRoll/goamz/s3/multi_test.go | 480 -------------- .../AdRoll/goamz/s3/responses_test.go | 248 ------- .../src/github.com/AdRoll/goamz/s3/s3_test.go | 513 --------------- .../github.com/AdRoll/goamz/s3/s3i_test.go | 603 ------------------ .../github.com/AdRoll/goamz/s3/s3t_test.go | 87 --- .../github.com/AdRoll/goamz/s3/sign_test.go | 148 ----- .../{AdRoll => docker}/goamz/aws/attempt.go | 0 .../{AdRoll => docker}/goamz/aws/aws.go | 0 .../{AdRoll => docker}/goamz/aws/client.go | 0 .../{AdRoll => docker}/goamz/aws/regions.go | 0 .../{AdRoll => docker}/goamz/aws/retry.go | 0 .../{AdRoll => docker}/goamz/aws/sign.go | 42 +- .../goamz/cloudfront/cloudfront.go | 2 +- .../{AdRoll => docker}/goamz/s3/lifecycle.go | 0 .../{AdRoll => docker}/goamz/s3/multi.go | 0 .../{AdRoll => docker}/goamz/s3/s3.go | 69 +- .../goamz/s3/s3test/server.go | 87 ++- .../{AdRoll => docker}/goamz/s3/sign.go | 2 +- registry/handlers/app.go | 55 +- .../middleware/cloudfront/middleware.go | 2 +- registry/storage/driver/s3/s3.go | 27 +- registry/storage/driver/s3/s3_test.go | 3 +- 32 files changed, 214 insertions(+), 3566 deletions(-) delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/aws/attempt_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/aws/export_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/aws/retry_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/cloudfront_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/testdata/key.pub delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/s3/export_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/s3/lifecycle_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/s3/responses_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3i_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3t_test.go delete mode 100644 Godeps/_workspace/src/github.com/AdRoll/goamz/s3/sign_test.go rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/aws/attempt.go (100%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/aws/aws.go (100%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/aws/client.go (100%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/aws/regions.go (100%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/aws/retry.go (100%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/aws/sign.go (93%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/cloudfront/cloudfront.go (99%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/s3/lifecycle.go (100%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/s3/multi.go (100%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/s3/s3.go (96%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/s3/s3test/server.go (94%) rename Godeps/_workspace/src/github.com/{AdRoll => docker}/goamz/s3/sign.go (98%) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 16100e21..72d4617a 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -41,18 +41,6 @@ "ImportPath": "google.golang.org/cloud/storage", "Rev": "2400193c85c3561d13880d34e0e10c4315bb02af" }, - { - "ImportPath": "github.com/AdRoll/goamz/aws", - "Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1" - }, - { - "ImportPath": "github.com/AdRoll/goamz/cloudfront", - "Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1" - }, - { - "ImportPath": "github.com/AdRoll/goamz/s3", - "Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1" - }, { "ImportPath": "github.com/Azure/azure-sdk-for-go/storage", "Rev": "97d9593768bbbbd316f9c055dfc5f780933cd7fc" @@ -87,6 +75,18 @@ "ImportPath": "github.com/denverdino/aliyungo/common", "Rev": "6ffb587da9da6d029d0ce517b85fecc82172d502" }, + { + "ImportPath": "github.com/docker/goamz/aws", + "Rev": "fb9c4c25c583d56a0544da8d1094294908c68ee8" + }, + { + "ImportPath": "github.com/docker/goamz/cloudfront", + "Rev": "fb9c4c25c583d56a0544da8d1094294908c68ee8" + }, + { + "ImportPath": "github.com/docker/goamz/s3", + "Rev": "fb9c4c25c583d56a0544da8d1094294908c68ee8" + }, { "ImportPath": "github.com/docker/libtrust", "Rev": "fa567046d9b14f6aa788882a950d69651d230b21" diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/attempt_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/attempt_test.go deleted file mode 100644 index a6a0afc9..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/attempt_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package aws_test - -import ( - "github.com/AdRoll/goamz/aws" - "gopkg.in/check.v1" - "time" -) - -func (S) TestAttemptTiming(c *check.C) { - testAttempt := aws.AttemptStrategy{ - Total: 0.25e9, - Delay: 0.1e9, - } - want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9} - got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing - t0 := time.Now() - for a := testAttempt.Start(); a.Next(); { - got = append(got, time.Now().Sub(t0)) - } - got = append(got, time.Now().Sub(t0)) - c.Assert(got, check.HasLen, len(want)) - const margin = 0.01e9 - for i, got := range want { - lo := want[i] - margin - hi := want[i] + margin - if got < lo || got > hi { - c.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds()) - } - } -} - -func (S) TestAttemptNextHasNext(c *check.C) { - a := aws.AttemptStrategy{}.Start() - c.Assert(a.Next(), check.Equals, true) - c.Assert(a.Next(), check.Equals, false) - - a = aws.AttemptStrategy{}.Start() - c.Assert(a.Next(), check.Equals, true) - c.Assert(a.HasNext(), check.Equals, false) - c.Assert(a.Next(), check.Equals, false) - - a = aws.AttemptStrategy{Total: 2e8}.Start() - c.Assert(a.Next(), check.Equals, true) - c.Assert(a.HasNext(), check.Equals, true) - time.Sleep(2e8) - c.Assert(a.HasNext(), check.Equals, true) - c.Assert(a.Next(), check.Equals, true) - c.Assert(a.Next(), check.Equals, false) - - a = aws.AttemptStrategy{Total: 1e8, Min: 2}.Start() - time.Sleep(1e8) - c.Assert(a.Next(), check.Equals, true) - c.Assert(a.HasNext(), check.Equals, true) - c.Assert(a.Next(), check.Equals, true) - c.Assert(a.HasNext(), check.Equals, false) - c.Assert(a.Next(), check.Equals, false) -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws_test.go deleted file mode 100644 index 0577f5c8..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package aws_test - -import ( - "github.com/AdRoll/goamz/aws" - "gopkg.in/check.v1" - "io/ioutil" - "os" - "strings" - "testing" - "time" -) - -func Test(t *testing.T) { - check.TestingT(t) -} - -var _ = check.Suite(&S{}) - -type S struct { - environ []string -} - -func (s *S) SetUpSuite(c *check.C) { - s.environ = os.Environ() -} - -func (s *S) TearDownTest(c *check.C) { - os.Clearenv() - for _, kv := range s.environ { - l := strings.SplitN(kv, "=", 2) - os.Setenv(l[0], l[1]) - } -} - -func (s *S) TestEnvAuthNoSecret(c *check.C) { - os.Clearenv() - _, err := aws.EnvAuth() - c.Assert(err, check.ErrorMatches, "AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment") -} - -func (s *S) TestEnvAuthNoAccess(c *check.C) { - os.Clearenv() - os.Setenv("AWS_SECRET_ACCESS_KEY", "foo") - _, err := aws.EnvAuth() - c.Assert(err, check.ErrorMatches, "AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment") -} - -func (s *S) TestEnvAuth(c *check.C) { - os.Clearenv() - os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") - os.Setenv("AWS_ACCESS_KEY_ID", "access") - auth, err := aws.EnvAuth() - c.Assert(err, check.IsNil) - c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestEnvAuthAlt(c *check.C) { - os.Clearenv() - os.Setenv("AWS_SECRET_KEY", "secret") - os.Setenv("AWS_ACCESS_KEY", "access") - auth, err := aws.EnvAuth() - c.Assert(err, check.IsNil) - c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestGetAuthStatic(c *check.C) { - exptdate := time.Now().Add(time.Hour) - auth, err := aws.GetAuth("access", "secret", "token", exptdate) - c.Assert(err, check.IsNil) - c.Assert(auth.AccessKey, check.Equals, "access") - c.Assert(auth.SecretKey, check.Equals, "secret") - c.Assert(auth.Token(), check.Equals, "token") - c.Assert(auth.Expiration(), check.Equals, exptdate) -} - -func (s *S) TestGetAuthEnv(c *check.C) { - os.Clearenv() - os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") - os.Setenv("AWS_ACCESS_KEY_ID", "access") - auth, err := aws.GetAuth("", "", "", time.Time{}) - c.Assert(err, check.IsNil) - c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"}) -} - -func (s *S) TestEncode(c *check.C) { - c.Assert(aws.Encode("foo"), check.Equals, "foo") - c.Assert(aws.Encode("/"), check.Equals, "%2F") -} - -func (s *S) TestRegionsAreNamed(c *check.C) { - for n, r := range aws.Regions { - c.Assert(n, check.Equals, r.Name) - } -} - -func (s *S) TestCredentialsFileAuth(c *check.C) { - file, err := ioutil.TempFile("", "creds") - - if err != nil { - c.Fatal(err) - } - - iniFile := ` - -[default] ; comment 123 -aws_access_key_id = keyid1 ;comment -aws_secret_access_key=key1 - - [profile2] - aws_access_key_id = keyid2 ;comment - aws_secret_access_key=key2 - aws_session_token=token1 - -` - _, err = file.WriteString(iniFile) - if err != nil { - c.Fatal(err) - } - - err = file.Close() - if err != nil { - c.Fatal(err) - } - - // check non-existant profile - _, err = aws.CredentialFileAuth(file.Name(), "no profile", 30*time.Minute) - c.Assert(err, check.Not(check.Equals), nil) - - defaultProfile, err := aws.CredentialFileAuth(file.Name(), "default", 30*time.Minute) - c.Assert(err, check.Equals, nil) - c.Assert(defaultProfile.AccessKey, check.Equals, "keyid1") - c.Assert(defaultProfile.SecretKey, check.Equals, "key1") - c.Assert(defaultProfile.Token(), check.Equals, "") - - profile2, err := aws.CredentialFileAuth(file.Name(), "profile2", 30*time.Minute) - c.Assert(err, check.Equals, nil) - c.Assert(profile2.AccessKey, check.Equals, "keyid2") - c.Assert(profile2.SecretKey, check.Equals, "key2") - c.Assert(profile2.Token(), check.Equals, "token1") -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/export_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/export_test.go deleted file mode 100644 index 5f4a9dd0..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/export_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package aws - -import ( - "net/http" - "time" -) - -// V4Signer: -// Exporting methods for testing - -func (s *V4Signer) RequestTime(req *http.Request) time.Time { - return s.requestTime(req) -} - -func (s *V4Signer) CanonicalRequest(req *http.Request) string { - return s.canonicalRequest(req, "") -} - -func (s *V4Signer) StringToSign(t time.Time, creq string) string { - return s.stringToSign(t, creq) -} - -func (s *V4Signer) Signature(t time.Time, sts string) string { - return s.signature(t, sts) -} - -func (s *V4Signer) Authorization(header http.Header, t time.Time, signature string) string { - return s.authorization(header, t, signature) -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/retry_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/retry_test.go deleted file mode 100644 index c1f10be4..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/retry_test.go +++ /dev/null @@ -1,303 +0,0 @@ -package aws - -import ( - "math/rand" - "net" - "net/http" - "testing" - "time" -) - -type testInput struct { - res *http.Response - err error - numRetries int -} - -type testResult struct { - shouldRetry bool - delay time.Duration -} - -type testCase struct { - input testInput - defaultResult testResult - dynamoDBResult testResult -} - -var testCases = []testCase{ - // Test nil fields - testCase{ - input: testInput{ - err: nil, - res: nil, - numRetries: 0, - }, - defaultResult: testResult{ - shouldRetry: false, - delay: 300 * time.Millisecond, - }, - dynamoDBResult: testResult{ - shouldRetry: false, - delay: 25 * time.Millisecond, - }, - }, - // Test 3 different throttling exceptions - testCase{ - input: testInput{ - err: &Error{ - Code: "Throttling", - }, - numRetries: 0, - }, - defaultResult: testResult{ - shouldRetry: true, - delay: 617165505 * time.Nanosecond, // account for randomness with known seed - }, - dynamoDBResult: testResult{ - shouldRetry: true, - delay: 25 * time.Millisecond, - }, - }, - testCase{ - input: testInput{ - err: &Error{ - Code: "ThrottlingException", - }, - numRetries: 0, - }, - defaultResult: testResult{ - shouldRetry: true, - delay: 579393152 * time.Nanosecond, // account for randomness with known seed - }, - dynamoDBResult: testResult{ - shouldRetry: true, - delay: 25 * time.Millisecond, - }, - }, - testCase{ - input: testInput{ - err: &Error{ - Code: "ProvisionedThroughputExceededException", - }, - numRetries: 1, - }, - defaultResult: testResult{ - shouldRetry: true, - delay: 1105991654 * time.Nanosecond, // account for randomness with known seed - }, - dynamoDBResult: testResult{ - shouldRetry: true, - delay: 50 * time.Millisecond, - }, - }, - // Test a fake throttling exception - testCase{ - input: testInput{ - err: &Error{ - Code: "MyMadeUpThrottlingCode", - }, - numRetries: 0, - }, - defaultResult: testResult{ - shouldRetry: false, - delay: 300 * time.Millisecond, - }, - dynamoDBResult: testResult{ - shouldRetry: false, - delay: 25 * time.Millisecond, - }, - }, - // Test 5xx errors - testCase{ - input: testInput{ - res: &http.Response{ - StatusCode: http.StatusInternalServerError, - }, - numRetries: 1, - }, - defaultResult: testResult{ - shouldRetry: true, - delay: 600 * time.Millisecond, - }, - dynamoDBResult: testResult{ - shouldRetry: true, - delay: 50 * time.Millisecond, - }, - }, - testCase{ - input: testInput{ - res: &http.Response{ - StatusCode: http.StatusServiceUnavailable, - }, - numRetries: 1, - }, - defaultResult: testResult{ - shouldRetry: true, - delay: 600 * time.Millisecond, - }, - dynamoDBResult: testResult{ - shouldRetry: true, - delay: 50 * time.Millisecond, - }, - }, - // Test a random 400 error - testCase{ - input: testInput{ - res: &http.Response{ - StatusCode: http.StatusNotFound, - }, - numRetries: 1, - }, - defaultResult: testResult{ - shouldRetry: false, - delay: 600 * time.Millisecond, - }, - dynamoDBResult: testResult{ - shouldRetry: false, - delay: 50 * time.Millisecond, - }, - }, - // Test a temporary net.Error - testCase{ - input: testInput{ - res: &http.Response{}, - err: &net.DNSError{ - IsTimeout: true, - }, - numRetries: 2, - }, - defaultResult: testResult{ - shouldRetry: true, - delay: 1200 * time.Millisecond, - }, - dynamoDBResult: testResult{ - shouldRetry: true, - delay: 100 * time.Millisecond, - }, - }, - // Test a non-temporary net.Error - testCase{ - input: testInput{ - res: &http.Response{}, - err: &net.DNSError{ - IsTimeout: false, - }, - numRetries: 3, - }, - defaultResult: testResult{ - shouldRetry: false, - delay: 2400 * time.Millisecond, - }, - dynamoDBResult: testResult{ - shouldRetry: false, - delay: 200 * time.Millisecond, - }, - }, - // Assert failure after hitting max default retries - testCase{ - input: testInput{ - err: &Error{ - Code: "ProvisionedThroughputExceededException", - }, - numRetries: defaultMaxRetries, - }, - defaultResult: testResult{ - shouldRetry: false, - delay: 4313582352 * time.Nanosecond, // account for randomness with known seed - }, - dynamoDBResult: testResult{ - shouldRetry: true, - delay: 200 * time.Millisecond, - }, - }, - // Assert failure after hitting max DynamoDB retries - testCase{ - input: testInput{ - err: &Error{ - Code: "ProvisionedThroughputExceededException", - }, - numRetries: dynamoDBMaxRetries, - }, - defaultResult: testResult{ - shouldRetry: false, - delay: maxDelay, - }, - dynamoDBResult: testResult{ - shouldRetry: false, - delay: maxDelay, - }, - }, - // Assert we never go over the maxDelay value - testCase{ - input: testInput{ - numRetries: 25, - }, - defaultResult: testResult{ - shouldRetry: false, - delay: maxDelay, - }, - dynamoDBResult: testResult{ - shouldRetry: false, - delay: maxDelay, - }, - }, -} - -func TestDefaultRetryPolicy(t *testing.T) { - rand.Seed(0) - var policy RetryPolicy - policy = &DefaultRetryPolicy{} - for _, test := range testCases { - res := test.input.res - err := test.input.err - numRetries := test.input.numRetries - - shouldRetry := policy.ShouldRetry("", res, err, numRetries) - if shouldRetry != test.defaultResult.shouldRetry { - t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, test.defaultResult.shouldRetry, res, err, numRetries) - } - delay := policy.Delay("", res, err, numRetries) - if delay != test.defaultResult.delay { - t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, test.defaultResult.delay, res, err, numRetries) - } - } -} - -func TestDynamoDBRetryPolicy(t *testing.T) { - var policy RetryPolicy - policy = &DynamoDBRetryPolicy{} - for _, test := range testCases { - res := test.input.res - err := test.input.err - numRetries := test.input.numRetries - - shouldRetry := policy.ShouldRetry("", res, err, numRetries) - if shouldRetry != test.dynamoDBResult.shouldRetry { - t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, test.dynamoDBResult.shouldRetry, res, err, numRetries) - } - delay := policy.Delay("", res, err, numRetries) - if delay != test.dynamoDBResult.delay { - t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, test.dynamoDBResult.delay, res, err, numRetries) - } - } -} - -func TestNeverRetryPolicy(t *testing.T) { - var policy RetryPolicy - policy = &NeverRetryPolicy{} - for _, test := range testCases { - res := test.input.res - err := test.input.err - numRetries := test.input.numRetries - - shouldRetry := policy.ShouldRetry("", res, err, numRetries) - if shouldRetry { - t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, false, res, err, numRetries) - } - delay := policy.Delay("", res, err, numRetries) - if delay != time.Duration(0) { - t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, time.Duration(0), res, err, numRetries) - } - } -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign_test.go deleted file mode 100644 index 0f01bce3..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign_test.go +++ /dev/null @@ -1,569 +0,0 @@ -package aws_test - -import ( - "fmt" - "github.com/AdRoll/goamz/aws" - "gopkg.in/check.v1" - "net/http" - "strings" - "time" -) - -var _ = check.Suite(&V4SignerSuite{}) - -type V4SignerSuite struct { - auth aws.Auth - region aws.Region - cases []V4SignerSuiteCase -} - -type V4SignerSuiteCase struct { - label string - request V4SignerSuiteCaseRequest - canonicalRequest string - stringToSign string - signature string - authorization string -} - -type V4SignerSuiteCaseRequest struct { - method string - host string - url string - headers []string - body string -} - -func (s *V4SignerSuite) SetUpSuite(c *check.C) { - s.auth = aws.Auth{AccessKey: "AKIDEXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"} - s.region = aws.USEast - - // Test cases from the Signature Version 4 Test Suite (http://goo.gl/nguvs0) - s.cases = append(s.cases, - - // get-header-key-duplicate - V4SignerSuiteCase{ - label: "get-header-key-duplicate", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "ZOO:zoobar", "zoo:foobar", "zoo:zoobar"}, - }, - canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:foobar,zoobar,zoobar\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n3c52f0eaae2b61329c0a332e3fa15842a37bc5812cf4d80eb64784308850e313", - signature: "54afcaaf45b331f81cd2edb974f7b824ff4dd594cbbaa945ed636b48477368ed", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=54afcaaf45b331f81cd2edb974f7b824ff4dd594cbbaa945ed636b48477368ed", - }, - - // get-header-value-order - V4SignerSuiteCase{ - label: "get-header-value-order", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "p:z", "p:a", "p:p", "p:a"}, - }, - canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\np:a,a,p,z\n\ndate;host;p\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n94c0389fefe0988cbbedc8606f0ca0b485b48da010d09fc844b45b697c8924fe", - signature: "d2973954263943b11624a11d1c963ca81fb274169c7868b2858c04f083199e3d", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;p, Signature=d2973954263943b11624a11d1c963ca81fb274169c7868b2858c04f083199e3d", - }, - - // get-header-value-trim - V4SignerSuiteCase{ - label: "get-header-value-trim", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "p: phfft "}, - }, - canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\np:phfft\n\ndate;host;p\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ndddd1902add08da1ac94782b05f9278c08dc7468db178a84f8950d93b30b1f35", - signature: "debf546796015d6f6ded8626f5ce98597c33b47b9164cf6b17b4642036fcb592", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;p, Signature=debf546796015d6f6ded8626f5ce98597c33b47b9164cf6b17b4642036fcb592", - }, - - // get-empty - V4SignerSuiteCase{ - label: "get-relative-relative", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // get-single-relative - V4SignerSuiteCase{ - label: "get-relative-relative", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/.", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // get-multiple-relative - V4SignerSuiteCase{ - label: "get-relative-relative", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/./././", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // get-relative-relative - V4SignerSuiteCase{ - label: "get-relative-relative", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/foo/bar/../..", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // get-relative - V4SignerSuiteCase{ - label: "get-relative", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/foo/..", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // get-slash-dot-slash - V4SignerSuiteCase{ - label: "get-slash-dot-slash", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/./", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // get-slash-pointless-dot - V4SignerSuiteCase{ - label: "get-slash-pointless-dot", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/./foo", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/foo\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n8021a97572ee460f87ca67f4e8c0db763216d84715f5424a843a5312a3321e2d", - signature: "910e4d6c9abafaf87898e1eb4c929135782ea25bb0279703146455745391e63a", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=910e4d6c9abafaf87898e1eb4c929135782ea25bb0279703146455745391e63a", - }, - - // get-slash - V4SignerSuiteCase{ - label: "get-slash", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "//", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // get-slashes - V4SignerSuiteCase{ - label: "get-slashes", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "//foo//", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/foo/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n6bb4476ee8745730c9cb79f33a0c70baa6d8af29c0077fa12e4e8f1dd17e7098", - signature: "b00392262853cfe3201e47ccf945601079e9b8a7f51ee4c3d9ee4f187aa9bf19", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b00392262853cfe3201e47ccf945601079e9b8a7f51ee4c3d9ee4f187aa9bf19", - }, - - // get-space - V4SignerSuiteCase{ - label: "get-space", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/%20/foo", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/%20/foo\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n69c45fb9fe3fd76442b5086e50b2e9fec8298358da957b293ef26e506fdfb54b", - signature: "f309cfbd10197a230c42dd17dbf5cca8a0722564cb40a872d25623cfa758e374", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=f309cfbd10197a230c42dd17dbf5cca8a0722564cb40a872d25623cfa758e374", - }, - - // get-unreserved - V4SignerSuiteCase{ - label: "get-unreserved", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ndf63ee3247c0356c696a3b21f8d8490b01fa9cd5bc6550ef5ef5f4636b7b8901", - signature: "830cc36d03f0f84e6ee4953fbe701c1c8b71a0372c63af9255aa364dd183281e", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=830cc36d03f0f84e6ee4953fbe701c1c8b71a0372c63af9255aa364dd183281e", - }, - - // get-utf8 - V4SignerSuiteCase{ - label: "get-utf8", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/%E1%88%B4", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/%E1%88%B4\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n27ba31df5dbc6e063d8f87d62eb07143f7f271c5330a917840586ac1c85b6f6b", - signature: "8d6634c189aa8c75c2e51e106b6b5121bed103fdb351f7d7d4381c738823af74", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=8d6634c189aa8c75c2e51e106b6b5121bed103fdb351f7d7d4381c738823af74", - }, - - // get-vanilla-empty-query-key - V4SignerSuiteCase{ - label: "get-vanilla-empty-query-key", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/?foo=bar", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n0846c2945b0832deb7a463c66af5c4f8bd54ec28c438e67a214445b157c9ddf8", - signature: "56c054473fd260c13e4e7393eb203662195f5d4a1fada5314b8b52b23f985e9f", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=56c054473fd260c13e4e7393eb203662195f5d4a1fada5314b8b52b23f985e9f", - }, - - // get-vanilla-query-order-key-case - V4SignerSuiteCase{ - label: "get-vanilla-query-order-key-case", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/?foo=Zoo&foo=aha", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\nfoo=Zoo&foo=aha\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ne25f777ba161a0f1baf778a87faf057187cf5987f17953320e3ca399feb5f00d", - signature: "be7148d34ebccdc6423b19085378aa0bee970bdc61d144bd1a8c48c33079ab09", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=be7148d34ebccdc6423b19085378aa0bee970bdc61d144bd1a8c48c33079ab09", - }, - - // get-vanilla-query-order-key - V4SignerSuiteCase{ - label: "get-vanilla-query-order-key", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/?a=foo&b=foo", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\na=foo&b=foo\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n2f23d14fe13caebf6dfda346285c6d9c14f49eaca8f5ec55c627dd7404f7a727", - signature: "0dc122f3b28b831ab48ba65cb47300de53fbe91b577fe113edac383730254a3b", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=0dc122f3b28b831ab48ba65cb47300de53fbe91b577fe113edac383730254a3b", - }, - - // get-vanilla-query-order-value - V4SignerSuiteCase{ - label: "get-vanilla-query-order-value", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/?foo=b&foo=a", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\nfoo=a&foo=b\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n33dffc220e89131f8f6157a35c40903daa658608d9129ff9489e5cf5bbd9b11b", - signature: "feb926e49e382bec75c9d7dcb2a1b6dc8aa50ca43c25d2bc51143768c0875acc", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=feb926e49e382bec75c9d7dcb2a1b6dc8aa50ca43c25d2bc51143768c0875acc", - }, - - // get-vanilla-query-unreserved - V4SignerSuiteCase{ - label: "get-vanilla-query-unreserved", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/?-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nd2578f3156d4c9d180713d1ff20601d8a3eed0dd35447d24603d7d67414bd6b5", - signature: "f1498ddb4d6dae767d97c466fb92f1b59a2c71ca29ac954692663f9db03426fb", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=f1498ddb4d6dae767d97c466fb92f1b59a2c71ca29ac954692663f9db03426fb", - }, - - // get-vanilla-query - V4SignerSuiteCase{ - label: "get-vanilla-query", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // get-vanilla-ut8-query - V4SignerSuiteCase{ - label: "get-vanilla-ut8-query", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/?ሴ=bar", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n%E1%88%B4=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nde5065ff39c131e6c2e2bd19cd9345a794bf3b561eab20b8d97b2093fc2a979e", - signature: "6fb359e9a05394cc7074e0feb42573a2601abc0c869a953e8c5c12e4e01f1a8c", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=6fb359e9a05394cc7074e0feb42573a2601abc0c869a953e8c5c12e4e01f1a8c", - }, - - // get-vanilla - V4SignerSuiteCase{ - label: "get-vanilla", - request: V4SignerSuiteCaseRequest{ - method: "GET", - host: "host.foo.com", - url: "/", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1", - signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470", - }, - - // post-header-key-case - V4SignerSuiteCase{ - label: "post-header-key-case", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n05da62cee468d24ae84faff3c39f1b85540de60243c1bcaace39c0a2acc7b2c4", - signature: "22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726", - }, - - // post-header-key-sort - V4SignerSuiteCase{ - label: "post-header-key-sort", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "ZOO:zoobar"}, - }, - canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:zoobar\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n34e1bddeb99e76ee01d63b5e28656111e210529efeec6cdfd46a48e4c734545d", - signature: "b7a95a52518abbca0964a999a880429ab734f35ebbf1235bd79a5de87756dc4a", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=b7a95a52518abbca0964a999a880429ab734f35ebbf1235bd79a5de87756dc4a", - }, - - // post-header-value-case - V4SignerSuiteCase{ - label: "post-header-value-case", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "zoo:ZOOBAR"}, - }, - canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:ZOOBAR\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n3aae6d8274b8c03e2cc96fc7d6bda4b9bd7a0a184309344470b2c96953e124aa", - signature: "273313af9d0c265c531e11db70bbd653f3ba074c1009239e8559d3987039cad7", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=273313af9d0c265c531e11db70bbd653f3ba074c1009239e8559d3987039cad7", - }, - - // post-vanilla-empty-query-value - V4SignerSuiteCase{ - label: "post-vanilla-empty-query-value", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/?foo=bar", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "POST\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ncd4f39132d8e60bb388831d734230460872b564871c47f5de62e62d1a68dbe1e", - signature: "b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92", - }, - - // post-vanilla-query - V4SignerSuiteCase{ - label: "post-vanilla-query", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/?foo=bar", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "POST\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ncd4f39132d8e60bb388831d734230460872b564871c47f5de62e62d1a68dbe1e", - signature: "b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92", - }, - - // post-vanilla - V4SignerSuiteCase{ - label: "post-vanilla", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - }, - canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n05da62cee468d24ae84faff3c39f1b85540de60243c1bcaace39c0a2acc7b2c4", - signature: "22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726", - }, - - // post-x-www-form-urlencoded-parameters - V4SignerSuiteCase{ - label: "post-x-www-form-urlencoded-parameters", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"Content-Type:application/x-www-form-urlencoded; charset=utf8", "Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - body: "foo=bar", - }, - canonicalRequest: "POST\n/\n\ncontent-type:application/x-www-form-urlencoded; charset=utf8\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ncontent-type;date;host\n3ba8907e7a252327488df390ed517c45b96dead033600219bdca7107d1d3f88a", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nc4115f9e54b5cecf192b1eaa23b8e88ed8dc5391bd4fde7b3fff3d9c9fe0af1f", - signature: "b105eb10c6d318d2294de9d49dd8b031b55e3c3fe139f2e637da70511e9e7b71", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=content-type;date;host, Signature=b105eb10c6d318d2294de9d49dd8b031b55e3c3fe139f2e637da70511e9e7b71", - }, - - // post-x-www-form-urlencoded - V4SignerSuiteCase{ - label: "post-x-www-form-urlencoded", - request: V4SignerSuiteCaseRequest{ - method: "POST", - host: "host.foo.com", - url: "/", - headers: []string{"Content-Type:application/x-www-form-urlencoded", "Date:Mon, 09 Sep 2011 23:36:00 GMT"}, - body: "foo=bar", - }, - canonicalRequest: "POST\n/\n\ncontent-type:application/x-www-form-urlencoded\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ncontent-type;date;host\n3ba8907e7a252327488df390ed517c45b96dead033600219bdca7107d1d3f88a", - stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n4c5c6e4b52fb5fb947a8733982a8a5a61b14f04345cbfe6e739236c76dd48f74", - signature: "5a15b22cf462f047318703b92e6f4f38884e4a7ab7b1d6426ca46a8bd1c26cbc", - authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=content-type;date;host, Signature=5a15b22cf462f047318703b92e6f4f38884e4a7ab7b1d6426ca46a8bd1c26cbc", - }, - ) -} - -func (s *V4SignerSuite) TestCases(c *check.C) { - signer := aws.NewV4Signer(s.auth, "host", s.region) - - for _, testCase := range s.cases { - - req, err := http.NewRequest(testCase.request.method, "http://"+testCase.request.host+testCase.request.url, strings.NewReader(testCase.request.body)) - c.Assert(err, check.IsNil, check.Commentf("Testcase: %s", testCase.label)) - for _, v := range testCase.request.headers { - h := strings.SplitN(v, ":", 2) - req.Header.Add(h[0], h[1]) - } - req.Header.Set("host", req.Host) - - t := signer.RequestTime(req) - - canonicalRequest := signer.CanonicalRequest(req) - c.Check(canonicalRequest, check.Equals, testCase.canonicalRequest, check.Commentf("Testcase: %s", testCase.label)) - - stringToSign := signer.StringToSign(t, canonicalRequest) - c.Check(stringToSign, check.Equals, testCase.stringToSign, check.Commentf("Testcase: %s", testCase.label)) - - signature := signer.Signature(t, stringToSign) - c.Check(signature, check.Equals, testCase.signature, check.Commentf("Testcase: %s", testCase.label)) - - authorization := signer.Authorization(req.Header, t, signature) - c.Check(authorization, check.Equals, testCase.authorization, check.Commentf("Testcase: %s", testCase.label)) - - signer.Sign(req) - c.Check(req.Header.Get("Authorization"), check.Equals, testCase.authorization, check.Commentf("Testcase: %s", testCase.label)) - } -} - -func ExampleV4Signer() { - // Get auth from env vars - auth, err := aws.EnvAuth() - if err != nil { - fmt.Println(err) - } - - // Create a signer with the auth, name of the service, and aws region - signer := aws.NewV4Signer(auth, "dynamodb", aws.USEast) - - // Create a request - req, err := http.NewRequest("POST", aws.USEast.DynamoDBEndpoint, strings.NewReader("sample_request")) - if err != nil { - fmt.Println(err) - } - - // Date or x-amz-date header is required to sign a request - req.Header.Add("Date", time.Now().UTC().Format(http.TimeFormat)) - - // Sign the request - signer.Sign(req) - - // Issue signed request - http.DefaultClient.Do(req) -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/cloudfront_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/cloudfront_test.go deleted file mode 100644 index 63744d1c..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/cloudfront_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package cloudfront - -import ( - "crypto/x509" - "encoding/pem" - "io/ioutil" - "net/url" - "testing" - "time" -) - -func TestSignedCannedURL(t *testing.T) { - rawKey, err := ioutil.ReadFile("testdata/key.pem") - if err != nil { - t.Fatal(err) - } - - pemKey, _ := pem.Decode(rawKey) - privateKey, err := x509.ParsePKCS1PrivateKey(pemKey.Bytes) - if err != nil { - t.Fatal(err) - } - - cf := &CloudFront{ - key: privateKey, - keyPairId: "test-key-pair-1231245", - BaseURL: "https://cloudfront.com", - } - - expireTime, err := time.Parse(time.RFC3339, "2014-03-28T14:00:21Z") - if err != nil { - t.Fatal(err) - } - - query := make(url.Values) - query.Add("test", "value") - - uri, err := cf.CannedSignedURL("test", "test=value", expireTime) - if err != nil { - t.Fatal(err) - } - - parsed, err := url.Parse(uri) - if err != nil { - t.Fatal(err) - } - - signature := parsed.Query().Get("Signature") - if signature == "" { - t.Fatal("Encoded signature is empty") - } -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/testdata/key.pub b/Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/testdata/key.pub deleted file mode 100644 index 7d0b5b4d..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/testdata/key.pub +++ /dev/null @@ -1,6 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC0yMzp9DkPAE99DhsEaGkqougL -vtmDKri4bZj0fFjmGmjyyjz9hlrsr87LHVWzH/7igK7040HG1UqypX3ijtJa9+6B -KHwBBctboU3y4GfwFwVAOumY9UytFpyPlgUFrffZLQAywKkT24OgcfEj0G5kiQn7 -60wFnmSUtOuITo708QIDAQAB ------END PUBLIC KEY----- diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/export_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/export_test.go deleted file mode 100644 index 80f62558..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/export_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package s3 - -import ( - "github.com/AdRoll/goamz/aws" -) - -var originalStrategy = attempts - -func SetAttemptStrategy(s *aws.AttemptStrategy) { - if s == nil { - attempts = originalStrategy - } else { - attempts = *s - } -} - -func Sign(auth aws.Auth, method, path string, params, headers map[string][]string) { - sign(auth, method, path, params, headers) -} - -func SetListPartsMax(n int) { - listPartsMax = n -} - -func SetListMultiMax(n int) { - listMultiMax = n -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/lifecycle_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/lifecycle_test.go deleted file mode 100644 index e43acb8f..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/lifecycle_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package s3_test - -import ( - "encoding/xml" - "github.com/AdRoll/goamz/s3" - "gopkg.in/check.v1" - "io/ioutil" - "net/http" - "strings" - "time" -) - -func (s *S) TestLifecycleConfiguration(c *check.C) { - date, err := time.Parse(s3.LifecycleRuleDateFormat, "2014-09-10") - c.Check(err, check.IsNil) - - conf := &s3.LifecycleConfiguration{} - - rule := s3.NewLifecycleRule("transition-days", "/") - rule.SetTransitionDays(7) - conf.AddRule(rule) - - rule = s3.NewLifecycleRule("transition-date", "/") - rule.SetTransitionDate(date) - conf.AddRule(rule) - - rule = s3.NewLifecycleRule("expiration-days", "") - rule.SetExpirationDays(1) - conf.AddRule(rule) - - rule = s3.NewLifecycleRule("expiration-date", "") - rule.SetExpirationDate(date) - conf.AddRule(rule) - - rule = s3.NewLifecycleRule("noncurrent-transition", "") - rule.SetNoncurrentVersionTransitionDays(11) - conf.AddRule(rule) - - rule = s3.NewLifecycleRule("noncurrent-expiration", "") - rule.SetNoncurrentVersionExpirationDays(1011) - - // Test Disable() and Enable() toggling - c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusEnabled) - rule.Disable() - c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusDisabled) - rule.Enable() - c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusEnabled) - rule.Disable() - c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusDisabled) - - conf.AddRule(rule) - - doc, err := xml.MarshalIndent(conf, "", " ") - c.Check(err, check.IsNil) - - expectedDoc := ` - - transition-days - / - Enabled - - 7 - GLACIER - - - - transition-date - / - Enabled - - 2014-09-10 - GLACIER - - - - expiration-days - - Enabled - - 1 - - - - expiration-date - - Enabled - - 2014-09-10 - - - - noncurrent-transition - - Enabled - - 11 - GLACIER - - - - noncurrent-expiration - - Disabled - - 1011 - - -` - - c.Check(string(doc), check.Equals, expectedDoc) - - // Unmarshalling test - conf2 := &s3.LifecycleConfiguration{} - err = xml.Unmarshal(doc, conf2) - c.Check(err, check.IsNil) - s.checkLifecycleConfigurationEqual(c, conf, conf2) -} - -func (s *S) checkLifecycleConfigurationEqual(c *check.C, conf, conf2 *s3.LifecycleConfiguration) { - c.Check(len(*conf2.Rules), check.Equals, len(*conf.Rules)) - for i, rule := range *conf2.Rules { - confRules := *conf.Rules - c.Check(rule, check.DeepEquals, confRules[i]) - } -} - -func (s *S) checkLifecycleRequest(c *check.C, req *http.Request) { - // ?lifecycle= is the only query param - v, ok := req.Form["lifecycle"] - c.Assert(ok, check.Equals, true) - c.Assert(v, check.HasLen, 1) - c.Assert(v[0], check.Equals, "") - - c.Assert(req.Header["X-Amz-Date"], check.HasLen, 1) - c.Assert(req.Header["X-Amz-Date"][0], check.Not(check.Equals), "") - - // Lifecycle methods require V4 auth - usesV4 := strings.HasPrefix(req.Header["Authorization"][0], "AWS4-HMAC-SHA256") - c.Assert(usesV4, check.Equals, true) -} - -func (s *S) TestPutLifecycleConfiguration(c *check.C) { - testServer.Response(200, nil, "") - - conf := &s3.LifecycleConfiguration{} - rule := s3.NewLifecycleRule("id", "") - rule.SetTransitionDays(7) - conf.AddRule(rule) - - doc, err := xml.Marshal(conf) - c.Check(err, check.IsNil) - - b := s.s3.Bucket("bucket") - err = b.PutLifecycleConfiguration(conf) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/bucket/") - c.Assert(req.Header["Content-Md5"], check.HasLen, 1) - c.Assert(req.Header["Content-Md5"][0], check.Not(check.Equals), "") - s.checkLifecycleRequest(c, req) - - // Check we sent the correct xml serialization - data, err := ioutil.ReadAll(req.Body) - req.Body.Close() - c.Assert(err, check.IsNil) - header := "\n" - c.Assert(string(data), check.Equals, header+string(doc)) -} - -func (s *S) TestGetLifecycleConfiguration(c *check.C) { - conf := &s3.LifecycleConfiguration{} - rule := s3.NewLifecycleRule("id", "") - rule.SetTransitionDays(7) - conf.AddRule(rule) - - doc, err := xml.Marshal(conf) - c.Check(err, check.IsNil) - - testServer.Response(200, nil, string(doc)) - - b := s.s3.Bucket("bucket") - conf2, err := b.GetLifecycleConfiguration() - c.Check(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/bucket/") - s.checkLifecycleRequest(c, req) - s.checkLifecycleConfigurationEqual(c, conf, conf2) -} - -func (s *S) TestDeleteLifecycleConfiguration(c *check.C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - err := b.DeleteLifecycleConfiguration() - c.Check(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "DELETE") - c.Assert(req.URL.Path, check.Equals, "/bucket/") - s.checkLifecycleRequest(c, req) -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go deleted file mode 100644 index b76f70eb..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go +++ /dev/null @@ -1,480 +0,0 @@ -package s3_test - -import ( - "encoding/xml" - "io" - "io/ioutil" - "strings" - - "github.com/AdRoll/goamz/s3" - "gopkg.in/check.v1" -) - -func (s *S) TestInitMulti(c *check.C) { - testServer.Response(200, nil, InitMultiResultDump) - b := s.s3.Bucket("sample") - - metadata := make(map[string][]string) - metadata["key1"] = []string{"value1"} - metadata["key2"] = []string{"value2"} - options := s3.Options{ - SSE: true, - Meta: metadata, - ContentEncoding: "text/utf8", - CacheControl: "no-cache", - RedirectLocation: "http://github.com/AdRoll/goamz", - ContentMD5: "0000000000000000", - } - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, options) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "POST") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"text/plain"}) - c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"}) - c.Assert(req.Form["uploads"], check.DeepEquals, []string{""}) - - c.Assert(req.Header["X-Amz-Server-Side-Encryption"], check.DeepEquals, []string{"AES256"}) - c.Assert(req.Header["Content-Encoding"], check.DeepEquals, []string{"text/utf8"}) - c.Assert(req.Header["Cache-Control"], check.DeepEquals, []string{"no-cache"}) - c.Assert(req.Header["Content-Md5"], check.DeepEquals, []string{"0000000000000000"}) - c.Assert(req.Header["X-Amz-Website-Redirect-Location"], check.DeepEquals, []string{"http://github.com/AdRoll/goamz"}) - c.Assert(req.Header["X-Amz-Meta-Key1"], check.DeepEquals, []string{"value1"}) - c.Assert(req.Header["X-Amz-Meta-Key2"], check.DeepEquals, []string{"value2"}) - - c.Assert(multi.UploadId, check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") -} - -func (s *S) TestMultiNoPreviousUpload(c *check.C) { - // Don't retry the NoSuchUpload error. - s.DisableRetries() - - testServer.Response(404, nil, NoSuchUploadErrorDump) - testServer.Response(200, nil, InitMultiResultDump) - - b := s.s3.Bucket("sample") - - multi, err := b.Multi("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/sample/") - c.Assert(req.Form["uploads"], check.DeepEquals, []string{""}) - c.Assert(req.Form["prefix"], check.DeepEquals, []string{"multi"}) - - req = testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "POST") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form["uploads"], check.DeepEquals, []string{""}) - - c.Assert(multi.UploadId, check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") -} - -func (s *S) TestMultiReturnOld(c *check.C) { - testServer.Response(200, nil, ListMultiResultDump) - - b := s.s3.Bucket("sample") - - multi, err := b.Multi("multi1", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - c.Assert(multi.Key, check.Equals, "multi1") - c.Assert(multi.UploadId, check.Equals, "iUVug89pPvSswrikD") - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/sample/") - c.Assert(req.Form["uploads"], check.DeepEquals, []string{""}) - c.Assert(req.Form["prefix"], check.DeepEquals, []string{"multi1"}) -} - -func (s *S) TestListParts(c *check.C) { - testServer.Response(200, nil, InitMultiResultDump) - testServer.Response(200, nil, ListPartsResultDump1) - testServer.Response(404, nil, NoSuchUploadErrorDump) // :-( - testServer.Response(200, nil, ListPartsResultDump2) - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - parts, err := multi.ListParts() - c.Assert(err, check.IsNil) - c.Assert(parts, check.HasLen, 3) - c.Assert(parts[0].N, check.Equals, 1) - c.Assert(parts[0].Size, check.Equals, int64(5)) - c.Assert(parts[0].ETag, check.Equals, `"ffc88b4ca90a355f8ddba6b2c3b2af5c"`) - c.Assert(parts[1].N, check.Equals, 2) - c.Assert(parts[1].Size, check.Equals, int64(5)) - c.Assert(parts[1].ETag, check.Equals, `"d067a0fa9dc61a6e7195ca99696b5a89"`) - c.Assert(parts[2].N, check.Equals, 3) - c.Assert(parts[2].Size, check.Equals, int64(5)) - c.Assert(parts[2].ETag, check.Equals, `"49dcd91231f801159e893fb5c6674985"`) - testServer.WaitRequest() - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") - c.Assert(req.Form["max-parts"], check.DeepEquals, []string{"1000"}) - - testServer.WaitRequest() // The internal error. - req = testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") - c.Assert(req.Form["max-parts"], check.DeepEquals, []string{"1000"}) - c.Assert(req.Form["part-number-marker"], check.DeepEquals, []string{"2"}) -} - -func (s *S) TestPutPart(c *check.C) { - headers := map[string]string{ - "ETag": `"26f90efd10d614f100252ff56d88dad8"`, - } - testServer.Response(200, nil, InitMultiResultDump) - testServer.Response(200, headers, "") - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - part, err := multi.PutPart(1, strings.NewReader("")) - c.Assert(err, check.IsNil) - c.Assert(part.N, check.Equals, 1) - c.Assert(part.Size, check.Equals, int64(8)) - c.Assert(part.ETag, check.Equals, headers["ETag"]) - - testServer.WaitRequest() - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") - c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"8"}) - c.Assert(req.Header["Content-Md5"], check.DeepEquals, []string{"JvkO/RDWFPEAJS/1bYja2A=="}) -} - -func (s *S) TestPutPartCopy(c *check.C) { - testServer.Response(200, nil, InitMultiResultDump) - // PutPartCopy makes a Head request internally to verify access to the source object - // and obtain its size - testServer.Response(200, nil, "content") - testServer.Response(200, nil, PutCopyResultDump) - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - res, part, err := multi.PutPartCopy(1, s3.CopyOptions{}, "source-bucket/\u00FCber-fil\u00E9.jpg") - c.Assert(err, check.IsNil) - c.Assert(part.N, check.Equals, 1) - c.Assert(part.Size, check.Equals, int64(7)) - c.Assert(res, check.DeepEquals, &s3.CopyObjectResult{ - ETag: `"9b2cf535f27731c974343645a3985328"`, - LastModified: `2009-10-28T22:32:00`}) - - // Verify the Head request - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "POST") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") - c.Assert(err, check.IsNil) - - testServer.WaitRequest() - req = testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") - c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"}) - c.Assert(req.Header["X-Amz-Copy-Source"], check.DeepEquals, []string{`source-bucket%2F%C3%BCber-fil%C3%A9.jpg`}) -} - -func readAll(r io.Reader) string { - data, err := ioutil.ReadAll(r) - if err != nil { - panic(err) - } - return string(data) -} - -func (s *S) TestPutAllNoPreviousUpload(c *check.C) { - // Don't retry the NoSuchUpload error. - s.DisableRetries() - - etag1 := map[string]string{"ETag": `"etag1"`} - etag2 := map[string]string{"ETag": `"etag2"`} - etag3 := map[string]string{"ETag": `"etag3"`} - testServer.Response(200, nil, InitMultiResultDump) - testServer.Response(404, nil, NoSuchUploadErrorDump) - testServer.Response(200, etag1, "") - testServer.Response(200, etag2, "") - testServer.Response(200, etag3, "") - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - parts, err := multi.PutAll(strings.NewReader("part1part2last"), 5) - c.Assert(parts, check.HasLen, 3) - c.Assert(parts[0].ETag, check.Equals, `"etag1"`) - c.Assert(parts[1].ETag, check.Equals, `"etag2"`) - c.Assert(parts[2].ETag, check.Equals, `"etag3"`) - c.Assert(err, check.IsNil) - - // Init - testServer.WaitRequest() - - // List old parts. Won't find anything. - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - - // Send part 1. - req = testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"}) - c.Assert(readAll(req.Body), check.Equals, "part1") - - // Send part 2. - req = testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"2"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"}) - c.Assert(readAll(req.Body), check.Equals, "part2") - - // Send part 3 with shorter body. - req = testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"3"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"4"}) - c.Assert(readAll(req.Body), check.Equals, "last") -} - -func (s *S) TestPutAllZeroSizeFile(c *check.C) { - // Don't retry the NoSuchUpload error. - s.DisableRetries() - - etag1 := map[string]string{"ETag": `"etag1"`} - testServer.Response(200, nil, InitMultiResultDump) - testServer.Response(404, nil, NoSuchUploadErrorDump) - testServer.Response(200, etag1, "") - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - // Must send at least one part, so that completing it will work. - parts, err := multi.PutAll(strings.NewReader(""), 5) - c.Assert(parts, check.HasLen, 1) - c.Assert(parts[0].ETag, check.Equals, `"etag1"`) - c.Assert(err, check.IsNil) - - // Init - testServer.WaitRequest() - - // List old parts. Won't find anything. - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - - // Send empty part. - req = testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"0"}) - c.Assert(readAll(req.Body), check.Equals, "") -} - -func (s *S) TestPutAllResume(c *check.C) { - etag2 := map[string]string{"ETag": `"etag2"`} - testServer.Response(200, nil, InitMultiResultDump) - testServer.Response(200, nil, ListPartsResultDump1) - testServer.Response(200, nil, ListPartsResultDump2) - testServer.Response(200, etag2, "") - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - // "part1" and "part3" match the checksums in ResultDump1. - // The middle one is a mismatch (it refers to "part2"). - parts, err := multi.PutAll(strings.NewReader("part1partXpart3"), 5) - c.Assert(parts, check.HasLen, 3) - c.Assert(parts[0].N, check.Equals, 1) - c.Assert(parts[0].Size, check.Equals, int64(5)) - c.Assert(parts[0].ETag, check.Equals, `"ffc88b4ca90a355f8ddba6b2c3b2af5c"`) - c.Assert(parts[1].N, check.Equals, 2) - c.Assert(parts[1].Size, check.Equals, int64(5)) - c.Assert(parts[1].ETag, check.Equals, `"etag2"`) - c.Assert(parts[2].N, check.Equals, 3) - c.Assert(parts[2].Size, check.Equals, int64(5)) - c.Assert(parts[2].ETag, check.Equals, `"49dcd91231f801159e893fb5c6674985"`) - c.Assert(err, check.IsNil) - - // Init - testServer.WaitRequest() - - // List old parts, broken in two requests. - for i := 0; i < 2; i++ { - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - } - - // Send part 2, as it didn't match the checksum. - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"2"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"}) - c.Assert(readAll(req.Body), check.Equals, "partX") -} - -func (s *S) TestMultiComplete(c *check.C) { - testServer.Response(200, nil, InitMultiResultDump) - testServer.Response(200, nil, MultiCompleteDump) - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}}) - c.Assert(err, check.IsNil) - - testServer.WaitRequest() - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "POST") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") - - var payload struct { - XMLName xml.Name - Part []struct { - PartNumber int - ETag string - } - } - - dec := xml.NewDecoder(req.Body) - err = dec.Decode(&payload) - c.Assert(err, check.IsNil) - - c.Assert(payload.XMLName.Local, check.Equals, "CompleteMultipartUpload") - c.Assert(len(payload.Part), check.Equals, 2) - c.Assert(payload.Part[0].PartNumber, check.Equals, 1) - c.Assert(payload.Part[0].ETag, check.Equals, `"ETag1"`) - c.Assert(payload.Part[1].PartNumber, check.Equals, 2) - c.Assert(payload.Part[1].ETag, check.Equals, `"ETag2"`) -} - -func (s *S) TestMultiCompleteError(c *check.C) { - testServer.Response(200, nil, InitMultiResultDump) - // Note the 200 response. Completing will hold the connection on some - // kind of long poll, and may return a late error even after a 200. - testServer.Response(200, nil, InternalErrorDump) - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}}) - c.Assert(err, check.NotNil) - - testServer.WaitRequest() - testServer.WaitRequest() -} - -func (s *S) TestMultiAbort(c *check.C) { - testServer.Response(200, nil, InitMultiResultDump) - testServer.Response(200, nil, "") - - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - err = multi.Abort() - c.Assert(err, check.IsNil) - - testServer.WaitRequest() - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "DELETE") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") -} - -func (s *S) TestListMulti(c *check.C) { - testServer.Response(200, nil, ListMultiResultDump) - - b := s.s3.Bucket("sample") - - multis, prefixes, err := b.ListMulti("", "/") - c.Assert(err, check.IsNil) - c.Assert(prefixes, check.DeepEquals, []string{"a/", "b/"}) - c.Assert(multis, check.HasLen, 2) - c.Assert(multis[0].Key, check.Equals, "multi1") - c.Assert(multis[0].UploadId, check.Equals, "iUVug89pPvSswrikD") - c.Assert(multis[1].Key, check.Equals, "multi2") - c.Assert(multis[1].UploadId, check.Equals, "DkirwsSvPp98guVUi") - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/sample/") - c.Assert(req.Form["uploads"], check.DeepEquals, []string{""}) - c.Assert(req.Form["prefix"], check.DeepEquals, []string{""}) - c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"}) - c.Assert(req.Form["max-uploads"], check.DeepEquals, []string{"1000"}) -} - -func (s *S) TestMultiCompleteSupportRadosGW(c *check.C) { - testServer.Response(200, nil, InitMultiResultDump) - testServer.Response(200, nil, MultiCompleteDump) - s.s3.Region.Name = "generic" - b := s.s3.Bucket("sample") - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}}) - c.Assert(err, check.IsNil) - - testServer.WaitRequest() - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "POST") - c.Assert(req.URL.Path, check.Equals, "/sample/multi") - c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--") - c.Assert(req.Header["Content-Length"], check.NotNil) - - var payload struct { - XMLName xml.Name - Part []struct { - PartNumber int - ETag string - } - } - - dec := xml.NewDecoder(req.Body) - err = dec.Decode(&payload) - c.Assert(err, check.IsNil) - - c.Assert(payload.XMLName.Local, check.Equals, "CompleteMultipartUpload") - c.Assert(len(payload.Part), check.Equals, 2) - c.Assert(payload.Part[0].PartNumber, check.Equals, 1) - c.Assert(payload.Part[0].ETag, check.Equals, `"ETag1"`) - c.Assert(payload.Part[1].PartNumber, check.Equals, 2) - c.Assert(payload.Part[1].ETag, check.Equals, `"ETag2"`) -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/responses_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/responses_test.go deleted file mode 100644 index 66fe271b..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/responses_test.go +++ /dev/null @@ -1,248 +0,0 @@ -package s3_test - -var PutCopyResultDump = ` - - - 2009-10-28T22:32:00 - "9b2cf535f27731c974343645a3985328" - -` - -var GetObjectErrorDump = ` - -NoSuchBucketThe specified bucket does not exist -non-existent-bucket3F1B667FAD71C3D8 -L4ee/zrm1irFXY5F45fKXIRdOf9ktsKY/8TDVawuMK2jWRb1RF84i1uBzkdNqS5D -` - -var GetListResultDump1 = ` - - - quotes - N - false - - Nelson - 2006-01-01T12:00:00.000Z - "828ef3fdfa96f00ad9f27c383fc9ac7f" - 5 - STANDARD - - bcaf161ca5fb16fd081034f - webfile - - - - Neo - 2006-01-01T12:00:00.000Z - "828ef3fdfa96f00ad9f27c383fc9ac7f" - 4 - STANDARD - - bcaf1ffd86a5fb16fd081034f - webfile - - - -` - -var GetListResultDump2 = ` - - example-bucket - photos/2006/ - some-marker - 1000 - / - false - - - photos/2006/feb/ - - - photos/2006/jan/ - - -` - -var InitMultiResultDump = ` - - - sample - multi - JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ-- - -` - -var ListPartsResultDump1 = ` - - - sample - multi - JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ-- - - bb5c0f63b0b25f2d099c - joe - - - bb5c0f63b0b25f2d099c - joe - - STANDARD - 0 - 2 - 2 - true - - 1 - 2013-01-30T13:45:51.000Z - "ffc88b4ca90a355f8ddba6b2c3b2af5c" - 5 - - - 2 - 2013-01-30T13:45:52.000Z - "d067a0fa9dc61a6e7195ca99696b5a89" - 5 - - -` - -var ListPartsResultDump2 = ` - - - sample - multi - JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ-- - - bb5c0f63b0b25f2d099c - joe - - - bb5c0f63b0b25f2d099c - joe - - STANDARD - 2 - 3 - 2 - false - - 3 - 2013-01-30T13:46:50.000Z - "49dcd91231f801159e893fb5c6674985" - 5 - - -` - -var ListMultiResultDump = ` - - - goamz-test-bucket-us-east-1-akiajk3wyewhctyqbf7a - - - multi1 - iUVug89pPvSswrikD72p8uO62EzhNtpDxRmwC5WSiWDdK9SfzmDqe3xpP1kMWimyimSnz4uzFc3waVM5ufrKYQ-- - / - 1000 - false - - multi1 - iUVug89pPvSswrikD - - bb5c0f63b0b25f2d0 - gustavoniemeyer - - - bb5c0f63b0b25f2d0 - gustavoniemeyer - - STANDARD - 2013-01-30T18:15:47.000Z - - - multi2 - DkirwsSvPp98guVUi - - bb5c0f63b0b25f2d0 - joe - - - bb5c0f63b0b25f2d0 - joe - - STANDARD - 2013-01-30T18:15:47.000Z - - - a/ - - - b/ - - -` - -var NoSuchUploadErrorDump = ` - - - NoSuchUpload - Not relevant - sample - 3F1B667FAD71C3D8 - kjhwqk - -` - -var MultiCompleteDump = ` - - http://Example-Bucket.s3.amazonaws.com/Example-Object - Example-Bucket - Example-Object - "3858f62230ac3c915f300c664312c11f-9" - -` - -var InternalErrorDump = ` - - - InternalError - Not relevant - sample - 3F1B667FAD71C3D8 - kjhwqk - -` - -var GetServiceDump = ` - - - - bcaf1ffd86f461ca5fb16fd081034f - webfile - - - - quotes - 2006-02-03T16:45:09.000Z - - - samples - 2006-02-03T16:41:58.000Z - - - -` - -var GetLocationUsStandard = ` - - -` - -var GetLocationUsWest1 = ` - -us-west-1 -` - -var BucketWebsiteConfigurationDump = ` -example.com` diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go deleted file mode 100644 index 7a17474c..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go +++ /dev/null @@ -1,513 +0,0 @@ -package s3_test - -import ( - "bytes" - "io/ioutil" - "net/http" - "testing" - "time" - - "github.com/AdRoll/goamz/aws" - "github.com/AdRoll/goamz/s3" - "github.com/AdRoll/goamz/testutil" - "gopkg.in/check.v1" -) - -func Test(t *testing.T) { - check.TestingT(t) -} - -type S struct { - s3 *s3.S3 -} - -var _ = check.Suite(&S{}) - -var testServer = testutil.NewHTTPServer() - -func (s *S) SetUpSuite(c *check.C) { - testServer.Start() - auth := aws.Auth{AccessKey: "abc", SecretKey: "123"} - s.s3 = s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL}) -} - -func (s *S) TearDownSuite(c *check.C) { - s3.SetAttemptStrategy(nil) -} - -func (s *S) SetUpTest(c *check.C) { - attempts := aws.AttemptStrategy{ - Total: 300 * time.Millisecond, - Delay: 100 * time.Millisecond, - } - s3.SetAttemptStrategy(&attempts) -} - -func (s *S) TearDownTest(c *check.C) { - testServer.Flush() -} - -func (s *S) DisableRetries() { - s3.SetAttemptStrategy(&aws.AttemptStrategy{}) -} - -// PutBucket docs: http://goo.gl/kBTCu - -func (s *S) TestPutBucket(c *check.C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - err := b.PutBucket(s3.Private) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/bucket/") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") -} - -// PutBucketWebsite docs: http://goo.gl/TpRlUy - -func (s *S) TestPutBucketWebsite(c *check.C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - config := s3.WebsiteConfiguration{ - RedirectAllRequestsTo: &s3.RedirectAllRequestsTo{HostName: "example.com"}, - } - err := b.PutBucketWebsite(config) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - body, err := ioutil.ReadAll(req.Body) - req.Body.Close() - c.Assert(err, check.IsNil) - c.Assert(string(body), check.Equals, BucketWebsiteConfigurationDump) - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/bucket/") - c.Assert(req.URL.RawQuery, check.Equals, "website=") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") -} - -// Head docs: http://bit.ly/17K1ylI - -func (s *S) TestHead(c *check.C) { - testServer.Response(200, nil, "content") - - b := s.s3.Bucket("bucket") - resp, err := b.Head("name", nil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "HEAD") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") - - c.Assert(err, check.IsNil) - c.Assert(resp.ContentLength, check.FitsTypeOf, int64(0)) - c.Assert(resp, check.FitsTypeOf, &http.Response{}) -} - -// DeleteBucket docs: http://goo.gl/GoBrY - -func (s *S) TestDelBucket(c *check.C) { - testServer.Response(204, nil, "") - - b := s.s3.Bucket("bucket") - err := b.DelBucket() - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "DELETE") - c.Assert(req.URL.Path, check.Equals, "/bucket/") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") -} - -// GetObject docs: http://goo.gl/isCO7 - -func (s *S) TestGet(c *check.C) { - testServer.Response(200, nil, "content") - - b := s.s3.Bucket("bucket") - data, err := b.Get("name") - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") - - c.Assert(err, check.IsNil) - c.Assert(string(data), check.Equals, "content") -} - -func (s *S) TestGetWithPlus(c *check.C) { - testServer.Response(200, nil, "content") - - b := s.s3.Bucket("bucket") - _, err := b.Get("has+plus") - - req := testServer.WaitRequest() - c.Assert(err, check.IsNil) - c.Assert(req.RequestURI, check.Equals, "http://localhost:4444/bucket/has%2Bplus") -} - -func (s *S) TestURL(c *check.C) { - testServer.Response(200, nil, "content") - - b := s.s3.Bucket("bucket") - url := b.URL("name") - r, err := http.Get(url) - c.Assert(err, check.IsNil) - data, err := ioutil.ReadAll(r.Body) - r.Body.Close() - c.Assert(err, check.IsNil) - c.Assert(string(data), check.Equals, "content") - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") -} - -func (s *S) TestGetReader(c *check.C) { - testServer.Response(200, nil, "content") - - b := s.s3.Bucket("bucket") - rc, err := b.GetReader("name") - c.Assert(err, check.IsNil) - data, err := ioutil.ReadAll(rc) - rc.Close() - c.Assert(err, check.IsNil) - c.Assert(string(data), check.Equals, "content") - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") -} - -func (s *S) TestGetNotFound(c *check.C) { - for i := 0; i < 10; i++ { - testServer.Response(404, nil, GetObjectErrorDump) - } - - b := s.s3.Bucket("non-existent-bucket") - data, err := b.Get("non-existent") - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/non-existent-bucket/non-existent") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") - - s3err, _ := err.(*s3.Error) - c.Assert(s3err, check.NotNil) - c.Assert(s3err.StatusCode, check.Equals, 404) - c.Assert(s3err.BucketName, check.Equals, "non-existent-bucket") - c.Assert(s3err.RequestId, check.Equals, "3F1B667FAD71C3D8") - c.Assert(s3err.HostId, check.Equals, "L4ee/zrm1irFXY5F45fKXIRdOf9ktsKY/8TDVawuMK2jWRb1RF84i1uBzkdNqS5D") - c.Assert(s3err.Code, check.Equals, "NoSuchBucket") - c.Assert(s3err.Message, check.Equals, "The specified bucket does not exist") - c.Assert(s3err.Error(), check.Equals, "The specified bucket does not exist") - c.Assert(data, check.IsNil) -} - -// PutObject docs: http://goo.gl/FEBPD - -func (s *S) TestPutObject(c *check.C) { - testServer.Response(200, nil, "") - const DISPOSITION = "attachment; filename=\"0x1a2b3c.jpg\"" - - b := s.s3.Bucket("bucket") - err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{ContentDisposition: DISPOSITION}) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") - c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""}) - c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"}) - c.Assert(req.Header["Content-Disposition"], check.DeepEquals, []string{DISPOSITION}) - //c.Assert(req.Header["Content-MD5"], gocheck.DeepEquals, "...") - c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"}) -} - -func (s *S) TestPutObjectReducedRedundancy(c *check.C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{StorageClass: s3.ReducedRedundancy}) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") - c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""}) - c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"}) - c.Assert(req.Header["X-Amz-Storage-Class"], check.DeepEquals, []string{"REDUCED_REDUNDANCY"}) -} - -// PutCopy docs: http://goo.gl/mhEHtA -func (s *S) TestPutCopy(c *check.C) { - testServer.Response(200, nil, PutCopyResultDump) - - b := s.s3.Bucket("bucket") - res, err := b.PutCopy("name", s3.Private, s3.CopyOptions{}, - // 0xFC is ü - 0xE9 is é - "source-bucket/\u00FCber-fil\u00E9.jpg") - c.Assert(err, check.IsNil) - c.Assert(res, check.DeepEquals, &s3.CopyObjectResult{ - ETag: `"9b2cf535f27731c974343645a3985328"`, - LastModified: `2009-10-28T22:32:00`}) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") - c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"0"}) - c.Assert(req.Header["X-Amz-Copy-Source"], check.DeepEquals, []string{`source-bucket%2F%C3%BCber-fil%C3%A9.jpg`}) - c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"}) -} - -func (s *S) TestPutObjectReadTimeout(c *check.C) { - s.s3.ReadTimeout = 50 * time.Millisecond - defer func() { - s.s3.ReadTimeout = 0 - }() - - b := s.s3.Bucket("bucket") - err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{}) - - // Make sure that we get a timeout error. - c.Assert(err, check.NotNil) - - // Set the response after the request times out so that the next request will work. - testServer.Response(200, nil, "") - - // This time set the response within our timeout period so that we expect the call - // to return successfully. - go func() { - time.Sleep(25 * time.Millisecond) - testServer.Response(200, nil, "") - }() - err = b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) -} - -func (s *S) TestPutReader(c *check.C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - buf := bytes.NewBufferString("content") - err := b.PutReader("name", buf, int64(buf.Len()), "content-type", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "PUT") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") - c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""}) - c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"}) - c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"}) - //c.Assert(req.Header["Content-MD5"], gocheck.Equals, "...") - c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"}) -} - -// DelObject docs: http://goo.gl/APeTt - -func (s *S) TestDelObject(c *check.C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - err := b.Del("name") - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "DELETE") - c.Assert(req.URL.Path, check.Equals, "/bucket/name") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") -} - -func (s *S) TestDelMultiObjects(c *check.C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - objects := []s3.Object{s3.Object{Key: "test"}} - err := b.DelMulti(s3.Delete{ - Quiet: false, - Objects: objects, - }) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "POST") - c.Assert(req.URL.RawQuery, check.Equals, "delete=") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") - c.Assert(req.Header["Content-MD5"], check.Not(check.Equals), "") - c.Assert(req.Header["Content-Type"], check.Not(check.Equals), "") - c.Assert(req.ContentLength, check.Not(check.Equals), "") -} - -// Bucket List Objects docs: http://goo.gl/YjQTc - -func (s *S) TestList(c *check.C) { - testServer.Response(200, nil, GetListResultDump1) - - b := s.s3.Bucket("quotes") - - data, err := b.List("N", "", "", 0) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/quotes/") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") - c.Assert(req.Form["prefix"], check.DeepEquals, []string{"N"}) - c.Assert(req.Form["delimiter"], check.DeepEquals, []string{""}) - c.Assert(req.Form["marker"], check.DeepEquals, []string{""}) - c.Assert(req.Form["max-keys"], check.DeepEquals, []string(nil)) - - c.Assert(data.Name, check.Equals, "quotes") - c.Assert(data.Prefix, check.Equals, "N") - c.Assert(data.IsTruncated, check.Equals, false) - c.Assert(len(data.Contents), check.Equals, 2) - - c.Assert(data.Contents[0].Key, check.Equals, "Nelson") - c.Assert(data.Contents[0].LastModified, check.Equals, "2006-01-01T12:00:00.000Z") - c.Assert(data.Contents[0].ETag, check.Equals, `"828ef3fdfa96f00ad9f27c383fc9ac7f"`) - c.Assert(data.Contents[0].Size, check.Equals, int64(5)) - c.Assert(data.Contents[0].StorageClass, check.Equals, "STANDARD") - c.Assert(data.Contents[0].Owner.ID, check.Equals, "bcaf161ca5fb16fd081034f") - c.Assert(data.Contents[0].Owner.DisplayName, check.Equals, "webfile") - - c.Assert(data.Contents[1].Key, check.Equals, "Neo") - c.Assert(data.Contents[1].LastModified, check.Equals, "2006-01-01T12:00:00.000Z") - c.Assert(data.Contents[1].ETag, check.Equals, `"828ef3fdfa96f00ad9f27c383fc9ac7f"`) - c.Assert(data.Contents[1].Size, check.Equals, int64(4)) - c.Assert(data.Contents[1].StorageClass, check.Equals, "STANDARD") - c.Assert(data.Contents[1].Owner.ID, check.Equals, "bcaf1ffd86a5fb16fd081034f") - c.Assert(data.Contents[1].Owner.DisplayName, check.Equals, "webfile") -} - -func (s *S) TestListWithDelimiter(c *check.C) { - testServer.Response(200, nil, GetListResultDump2) - - b := s.s3.Bucket("quotes") - - data, err := b.List("photos/2006/", "/", "some-marker", 1000) - c.Assert(err, check.IsNil) - - req := testServer.WaitRequest() - c.Assert(req.Method, check.Equals, "GET") - c.Assert(req.URL.Path, check.Equals, "/quotes/") - c.Assert(req.Header["Date"], check.Not(check.Equals), "") - c.Assert(req.Form["prefix"], check.DeepEquals, []string{"photos/2006/"}) - c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"}) - c.Assert(req.Form["marker"], check.DeepEquals, []string{"some-marker"}) - c.Assert(req.Form["max-keys"], check.DeepEquals, []string{"1000"}) - - c.Assert(data.Name, check.Equals, "example-bucket") - c.Assert(data.Prefix, check.Equals, "photos/2006/") - c.Assert(data.Delimiter, check.Equals, "/") - c.Assert(data.Marker, check.Equals, "some-marker") - c.Assert(data.IsTruncated, check.Equals, false) - c.Assert(len(data.Contents), check.Equals, 0) - c.Assert(data.CommonPrefixes, check.DeepEquals, []string{"photos/2006/feb/", "photos/2006/jan/"}) -} - -func (s *S) TestExists(c *check.C) { - testServer.Response(200, nil, "") - - b := s.s3.Bucket("bucket") - result, err := b.Exists("name") - - req := testServer.WaitRequest() - - c.Assert(req.Method, check.Equals, "HEAD") - - c.Assert(err, check.IsNil) - c.Assert(result, check.Equals, true) -} - -func (s *S) TestExistsNotFound404(c *check.C) { - testServer.Response(404, nil, "") - - b := s.s3.Bucket("bucket") - result, err := b.Exists("name") - - req := testServer.WaitRequest() - - c.Assert(req.Method, check.Equals, "HEAD") - - c.Assert(err, check.IsNil) - c.Assert(result, check.Equals, false) -} - -func (s *S) TestExistsNotFound403(c *check.C) { - testServer.Response(403, nil, "") - - b := s.s3.Bucket("bucket") - result, err := b.Exists("name") - - req := testServer.WaitRequest() - - c.Assert(req.Method, check.Equals, "HEAD") - - c.Assert(err, check.IsNil) - c.Assert(result, check.Equals, false) -} - -func (s *S) TestGetService(c *check.C) { - testServer.Response(200, nil, GetServiceDump) - - expected := s3.GetServiceResp{ - Owner: s3.Owner{ - ID: "bcaf1ffd86f461ca5fb16fd081034f", - DisplayName: "webfile", - }, - Buckets: []s3.BucketInfo{ - s3.BucketInfo{ - Name: "quotes", - CreationDate: "2006-02-03T16:45:09.000Z", - }, - s3.BucketInfo{ - Name: "samples", - CreationDate: "2006-02-03T16:41:58.000Z", - }, - }, - } - - received, err := s.s3.GetService() - - c.Assert(err, check.IsNil) - c.Assert(*received, check.DeepEquals, expected) -} - -func (s *S) TestLocation(c *check.C) { - testServer.Response(200, nil, GetLocationUsStandard) - expectedUsStandard := "us-east-1" - - bucketUsStandard := s.s3.Bucket("us-east-1") - resultUsStandard, err := bucketUsStandard.Location() - - c.Assert(err, check.IsNil) - c.Assert(resultUsStandard, check.Equals, expectedUsStandard) - - testServer.Response(200, nil, GetLocationUsWest1) - expectedUsWest1 := "us-west-1" - - bucketUsWest1 := s.s3.Bucket("us-west-1") - resultUsWest1, err := bucketUsWest1.Location() - - c.Assert(err, check.IsNil) - c.Assert(resultUsWest1, check.Equals, expectedUsWest1) -} - -func (s *S) TestSupportRadosGW(c *check.C) { - testServer.Response(200, nil, "content") - s.s3.Region.Name = "generic" - b := s.s3.Bucket("bucket") - _, err := b.Get("rgw") - - req := testServer.WaitRequest() - c.Assert(err, check.IsNil) - c.Assert(req.RequestURI, check.Equals, "/bucket/rgw") -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3i_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3i_test.go deleted file mode 100644 index b0da0130..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3i_test.go +++ /dev/null @@ -1,603 +0,0 @@ -package s3_test - -import ( - "bytes" - "crypto/md5" - "fmt" - "github.com/AdRoll/goamz/aws" - "github.com/AdRoll/goamz/s3" - "github.com/AdRoll/goamz/testutil" - "gopkg.in/check.v1" - "io/ioutil" - "net" - "net/http" - "sort" - "strings" - "time" -) - -// AmazonServer represents an Amazon S3 server. -type AmazonServer struct { - auth aws.Auth -} - -func (s *AmazonServer) SetUp(c *check.C) { - auth, err := aws.EnvAuth() - if err != nil { - c.Fatal(err.Error()) - } - s.auth = auth -} - -var _ = check.Suite(&AmazonClientSuite{Region: aws.USEast}) -var _ = check.Suite(&AmazonClientSuite{Region: aws.EUWest}) -var _ = check.Suite(&AmazonDomainClientSuite{Region: aws.USEast}) - -// AmazonClientSuite tests the client against a live S3 server. -type AmazonClientSuite struct { - aws.Region - srv AmazonServer - ClientTests -} - -func (s *AmazonClientSuite) SetUpSuite(c *check.C) { - if !testutil.Amazon { - c.Skip("live tests against AWS disabled (no -amazon)") - } - s.srv.SetUp(c) - s.s3 = s3.New(s.srv.auth, s.Region) - // In case tests were interrupted in the middle before. - s.ClientTests.Cleanup() -} - -func (s *AmazonClientSuite) TearDownTest(c *check.C) { - s.ClientTests.Cleanup() -} - -// AmazonDomainClientSuite tests the client against a live S3 -// server using bucket names in the endpoint domain name rather -// than the request path. -type AmazonDomainClientSuite struct { - aws.Region - srv AmazonServer - ClientTests -} - -func (s *AmazonDomainClientSuite) SetUpSuite(c *check.C) { - if !testutil.Amazon { - c.Skip("live tests against AWS disabled (no -amazon)") - } - s.srv.SetUp(c) - region := s.Region - region.S3BucketEndpoint = "https://${bucket}.s3.amazonaws.com" - s.s3 = s3.New(s.srv.auth, region) - s.ClientTests.Cleanup() -} - -func (s *AmazonDomainClientSuite) TearDownTest(c *check.C) { - s.ClientTests.Cleanup() -} - -// ClientTests defines integration tests designed to test the client. -// It is not used as a test suite in itself, but embedded within -// another type. -type ClientTests struct { - s3 *s3.S3 - authIsBroken bool -} - -func (s *ClientTests) Cleanup() { - killBucket(testBucket(s.s3)) -} - -func testBucket(s *s3.S3) *s3.Bucket { - // Watch out! If this function is corrupted and made to match with something - // people own, killBucket will happily remove *everything* inside the bucket. - key := s.Auth.AccessKey - if len(key) >= 8 { - key = s.Auth.AccessKey[:8] - } - return s.Bucket(fmt.Sprintf("goamz-%s-%s", s.Region.Name, key)) -} - -var attempts = aws.AttemptStrategy{ - Min: 5, - Total: 20 * time.Second, - Delay: 100 * time.Millisecond, -} - -func killBucket(b *s3.Bucket) { - var err error - for attempt := attempts.Start(); attempt.Next(); { - err = b.DelBucket() - if err == nil { - return - } - if _, ok := err.(*net.DNSError); ok { - return - } - e, ok := err.(*s3.Error) - if ok && e.Code == "NoSuchBucket" { - return - } - if ok && e.Code == "BucketNotEmpty" { - // Errors are ignored here. Just retry. - resp, err := b.List("", "", "", 1000) - if err == nil { - for _, key := range resp.Contents { - _ = b.Del(key.Key) - } - } - multis, _, _ := b.ListMulti("", "") - for _, m := range multis { - _ = m.Abort() - } - } - } - message := "cannot delete test bucket" - if err != nil { - message += ": " + err.Error() - } - panic(message) -} - -func get(url string) ([]byte, error) { - for attempt := attempts.Start(); attempt.Next(); { - resp, err := http.Get(url) - if err != nil { - if attempt.HasNext() { - continue - } - return nil, err - } - data, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - if attempt.HasNext() { - continue - } - return nil, err - } - return data, err - } - panic("unreachable") -} - -func (s *ClientTests) TestBasicFunctionality(c *check.C) { - b := testBucket(s.s3) - err := b.PutBucket(s3.PublicRead) - c.Assert(err, check.IsNil) - - err = b.Put("name", []byte("yo!"), "text/plain", s3.PublicRead, s3.Options{}) - c.Assert(err, check.IsNil) - defer b.Del("name") - - data, err := b.Get("name") - c.Assert(err, check.IsNil) - c.Assert(string(data), check.Equals, "yo!") - - data, err = get(b.URL("name")) - c.Assert(err, check.IsNil) - c.Assert(string(data), check.Equals, "yo!") - - buf := bytes.NewBufferString("hey!") - err = b.PutReader("name2", buf, int64(buf.Len()), "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - defer b.Del("name2") - - rc, err := b.GetReader("name2") - c.Assert(err, check.IsNil) - data, err = ioutil.ReadAll(rc) - c.Check(err, check.IsNil) - c.Check(string(data), check.Equals, "hey!") - rc.Close() - - data, err = get(b.SignedURL("name2", time.Now().Add(time.Hour))) - c.Assert(err, check.IsNil) - c.Assert(string(data), check.Equals, "hey!") - - if !s.authIsBroken { - data, err = get(b.SignedURL("name2", time.Now().Add(-time.Hour))) - c.Assert(err, check.IsNil) - c.Assert(string(data), check.Matches, "(?s).*AccessDenied.*") - } - - err = b.DelBucket() - c.Assert(err, check.NotNil) - - s3err, ok := err.(*s3.Error) - c.Assert(ok, check.Equals, true) - c.Assert(s3err.Code, check.Equals, "BucketNotEmpty") - c.Assert(s3err.BucketName, check.Equals, b.Name) - c.Assert(s3err.Message, check.Equals, "The bucket you tried to delete is not empty") - - err = b.Del("name") - c.Assert(err, check.IsNil) - err = b.Del("name2") - c.Assert(err, check.IsNil) - - err = b.DelBucket() - c.Assert(err, check.IsNil) -} - -func (s *ClientTests) TestGetNotFound(c *check.C) { - b := s.s3.Bucket("goamz-" + s.s3.Auth.AccessKey) - data, err := b.Get("non-existent") - - s3err, _ := err.(*s3.Error) - c.Assert(s3err, check.NotNil) - c.Assert(s3err.StatusCode, check.Equals, 404) - c.Assert(s3err.Code, check.Equals, "NoSuchBucket") - c.Assert(s3err.Message, check.Equals, "The specified bucket does not exist") - c.Assert(data, check.IsNil) -} - -// Communicate with all endpoints to see if they are alive. -func (s *ClientTests) TestRegions(c *check.C) { - errs := make(chan error, len(aws.Regions)) - for _, region := range aws.Regions { - go func(r aws.Region) { - s := s3.New(s.s3.Auth, r) - b := s.Bucket("goamz-" + s.Auth.AccessKey) - _, err := b.Get("non-existent") - errs <- err - }(region) - } - for _ = range aws.Regions { - err := <-errs - if err != nil { - s3_err, ok := err.(*s3.Error) - if ok { - c.Check(s3_err.Code, check.Matches, "NoSuchBucket") - } else if _, ok = err.(*net.DNSError); ok { - // Okay as well. - } else { - c.Errorf("Non-S3 error: %s", err) - } - } else { - c.Errorf("Test should have errored but it seems to have succeeded") - } - } -} - -var objectNames = []string{ - "index.html", - "index2.html", - "photos/2006/February/sample2.jpg", - "photos/2006/February/sample3.jpg", - "photos/2006/February/sample4.jpg", - "photos/2006/January/sample.jpg", - "test/bar", - "test/foo", -} - -func keys(names ...string) []s3.Key { - ks := make([]s3.Key, len(names)) - for i, name := range names { - ks[i].Key = name - } - return ks -} - -// As the ListResp specifies all the parameters to the -// request too, we use it to specify request parameters -// and expected results. The Contents field is -// used only for the key names inside it. -var listTests = []s3.ListResp{ - // normal list. - { - Contents: keys(objectNames...), - }, { - Marker: objectNames[0], - Contents: keys(objectNames[1:]...), - }, { - Marker: objectNames[0] + "a", - Contents: keys(objectNames[1:]...), - }, { - Marker: "z", - }, - - // limited results. - { - MaxKeys: 2, - Contents: keys(objectNames[0:2]...), - IsTruncated: true, - }, { - MaxKeys: 2, - Marker: objectNames[0], - Contents: keys(objectNames[1:3]...), - IsTruncated: true, - }, { - MaxKeys: 2, - Marker: objectNames[len(objectNames)-2], - Contents: keys(objectNames[len(objectNames)-1:]...), - }, - - // with delimiter - { - Delimiter: "/", - CommonPrefixes: []string{"photos/", "test/"}, - Contents: keys("index.html", "index2.html"), - }, { - Delimiter: "/", - Prefix: "photos/2006/", - CommonPrefixes: []string{"photos/2006/February/", "photos/2006/January/"}, - }, { - Delimiter: "/", - Prefix: "t", - CommonPrefixes: []string{"test/"}, - }, { - Delimiter: "/", - MaxKeys: 1, - Contents: keys("index.html"), - IsTruncated: true, - }, { - Delimiter: "/", - MaxKeys: 1, - Marker: "index2.html", - CommonPrefixes: []string{"photos/"}, - IsTruncated: true, - }, { - Delimiter: "/", - MaxKeys: 1, - Marker: "photos/", - CommonPrefixes: []string{"test/"}, - IsTruncated: false, - }, { - Delimiter: "Feb", - CommonPrefixes: []string{"photos/2006/Feb"}, - Contents: keys("index.html", "index2.html", "photos/2006/January/sample.jpg", "test/bar", "test/foo"), - }, -} - -func (s *ClientTests) TestDoublePutBucket(c *check.C) { - b := testBucket(s.s3) - err := b.PutBucket(s3.PublicRead) - c.Assert(err, check.IsNil) - - err = b.PutBucket(s3.PublicRead) - if err != nil { - c.Assert(err, check.FitsTypeOf, new(s3.Error)) - c.Assert(err.(*s3.Error).Code, check.Equals, "BucketAlreadyOwnedByYou") - } -} - -func (s *ClientTests) TestBucketList(c *check.C) { - b := testBucket(s.s3) - err := b.PutBucket(s3.Private) - c.Assert(err, check.IsNil) - - objData := make(map[string][]byte) - for i, path := range objectNames { - data := []byte(strings.Repeat("a", i)) - err := b.Put(path, data, "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - defer b.Del(path) - objData[path] = data - } - - for i, t := range listTests { - c.Logf("test %d", i) - resp, err := b.List(t.Prefix, t.Delimiter, t.Marker, t.MaxKeys) - c.Assert(err, check.IsNil) - c.Check(resp.Name, check.Equals, b.Name) - c.Check(resp.Delimiter, check.Equals, t.Delimiter) - c.Check(resp.IsTruncated, check.Equals, t.IsTruncated) - c.Check(resp.CommonPrefixes, check.DeepEquals, t.CommonPrefixes) - checkContents(c, resp.Contents, objData, t.Contents) - } -} - -func etag(data []byte) string { - sum := md5.New() - sum.Write(data) - return fmt.Sprintf(`"%x"`, sum.Sum(nil)) -} - -func checkContents(c *check.C, contents []s3.Key, data map[string][]byte, expected []s3.Key) { - c.Assert(contents, check.HasLen, len(expected)) - for i, k := range contents { - c.Check(k.Key, check.Equals, expected[i].Key) - // TODO mtime - c.Check(k.Size, check.Equals, int64(len(data[k.Key]))) - c.Check(k.ETag, check.Equals, etag(data[k.Key])) - } -} - -func (s *ClientTests) TestMultiInitPutList(c *check.C) { - b := testBucket(s.s3) - err := b.PutBucket(s3.Private) - c.Assert(err, check.IsNil) - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - c.Assert(multi.UploadId, check.Matches, ".+") - defer multi.Abort() - - var sent []s3.Part - - for i := 0; i < 5; i++ { - p, err := multi.PutPart(i+1, strings.NewReader(fmt.Sprintf("", i+1))) - c.Assert(err, check.IsNil) - c.Assert(p.N, check.Equals, i+1) - c.Assert(p.Size, check.Equals, int64(8)) - c.Assert(p.ETag, check.Matches, ".+") - sent = append(sent, p) - } - - s3.SetListPartsMax(2) - - parts, err := multi.ListParts() - c.Assert(err, check.IsNil) - c.Assert(parts, check.HasLen, len(sent)) - for i := range parts { - c.Assert(parts[i].N, check.Equals, sent[i].N) - c.Assert(parts[i].Size, check.Equals, sent[i].Size) - c.Assert(parts[i].ETag, check.Equals, sent[i].ETag) - } - - err = multi.Complete(parts) - s3err, failed := err.(*s3.Error) - c.Assert(failed, check.Equals, true) - c.Assert(s3err.Code, check.Equals, "EntityTooSmall") - - err = multi.Abort() - c.Assert(err, check.IsNil) - _, err = multi.ListParts() - s3err, ok := err.(*s3.Error) - c.Assert(ok, check.Equals, true) - c.Assert(s3err.Code, check.Equals, "NoSuchUpload") -} - -// This may take a minute or more due to the minimum size accepted S3 -// on multipart upload parts. -func (s *ClientTests) TestMultiComplete(c *check.C) { - b := testBucket(s.s3) - err := b.PutBucket(s3.Private) - c.Assert(err, check.IsNil) - - contentType := "text/plain" - meta := make(map[string][]string) - meta["X-Amz-Meta-TestField"] = []string{"testValue"} - options := s3.Options{ContentEncoding: "identity", ContentDisposition: "inline", Meta: meta} - multi, err := b.InitMulti("multi", contentType, s3.Private, options) - c.Assert(err, check.IsNil) - c.Assert(multi.UploadId, check.Matches, ".+") - defer multi.Abort() - - // Minimum size S3 accepts for all but the last part is 5MB. - data1 := make([]byte, 5*1024*1024) - data2 := []byte("") - - part1, err := multi.PutPart(1, bytes.NewReader(data1)) - c.Assert(err, check.IsNil) - part2, err := multi.PutPart(2, bytes.NewReader(data2)) - c.Assert(err, check.IsNil) - - // Purposefully reversed. The order requirement must be handled. - err = multi.Complete([]s3.Part{part2, part1}) - c.Assert(err, check.IsNil) - - data, err := b.Get("multi") - c.Assert(err, check.IsNil) - - c.Assert(len(data), check.Equals, len(data1)+len(data2)) - for i := range data1 { - if data[i] != data1[i] { - c.Fatalf("uploaded object at byte %d: want %d, got %d", data1[i], data[i]) - } - } - c.Assert(string(data[len(data1):]), check.Equals, string(data2)) - - resp, err := b.GetResponse("multi") - c.Assert(resp.Header.Get("Content-Type"), check.Equals, contentType) - c.Assert(resp.Header.Get("x-amz-acl"), check.Equals, s3.Private) - c.Assert(resp.Header.Get("Content-MD5"), check.Equals, options.ContentMD5) - c.Assert(resp.Header.Get("Content-Encoding"), check.Equals, options.ContentEncoding) - c.Assert(resp.Header.Get("Content-Disposition"), check.Equals, options.ContentDisposition) - for k, values := range meta { - c.Assert(resp.Header.Get(k), check.Equals, strings.Join(values, ",")) - } -} - -type multiList []*s3.Multi - -func (l multiList) Len() int { return len(l) } -func (l multiList) Less(i, j int) bool { return l[i].Key < l[j].Key } -func (l multiList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } - -func (s *ClientTests) TestListMulti(c *check.C) { - b := testBucket(s.s3) - err := b.PutBucket(s3.Private) - c.Assert(err, check.IsNil) - - // Ensure an empty state before testing its behavior. - multis, _, err := b.ListMulti("", "") - for _, m := range multis { - err := m.Abort() - c.Assert(err, check.IsNil) - } - - keys := []string{ - "a/multi2", - "a/multi3", - "b/multi4", - "multi1", - } - for _, key := range keys { - m, err := b.InitMulti(key, "", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - defer m.Abort() - } - - // Amazon's implementation of the multiple-request listing for - // multipart uploads in progress seems broken in multiple ways. - // (next tokens are not provided, etc). - //s3.SetListMultiMax(2) - - multis, prefixes, err := b.ListMulti("", "") - c.Assert(err, check.IsNil) - for attempt := attempts.Start(); attempt.Next() && len(multis) < len(keys); { - multis, prefixes, err = b.ListMulti("", "") - c.Assert(err, check.IsNil) - } - sort.Sort(multiList(multis)) - c.Assert(prefixes, check.IsNil) - var gotKeys []string - for _, m := range multis { - gotKeys = append(gotKeys, m.Key) - } - c.Assert(gotKeys, check.DeepEquals, keys) - for _, m := range multis { - c.Assert(m.Bucket, check.Equals, b) - c.Assert(m.UploadId, check.Matches, ".+") - } - - multis, prefixes, err = b.ListMulti("", "/") - for attempt := attempts.Start(); attempt.Next() && len(prefixes) < 2; { - multis, prefixes, err = b.ListMulti("", "") - c.Assert(err, check.IsNil) - } - c.Assert(err, check.IsNil) - c.Assert(prefixes, check.DeepEquals, []string{"a/", "b/"}) - c.Assert(multis, check.HasLen, 1) - c.Assert(multis[0].Bucket, check.Equals, b) - c.Assert(multis[0].Key, check.Equals, "multi1") - c.Assert(multis[0].UploadId, check.Matches, ".+") - - for attempt := attempts.Start(); attempt.Next() && len(multis) < 2; { - multis, prefixes, err = b.ListMulti("", "") - c.Assert(err, check.IsNil) - } - multis, prefixes, err = b.ListMulti("a/", "/") - c.Assert(err, check.IsNil) - c.Assert(prefixes, check.IsNil) - c.Assert(multis, check.HasLen, 2) - c.Assert(multis[0].Bucket, check.Equals, b) - c.Assert(multis[0].Key, check.Equals, "a/multi2") - c.Assert(multis[0].UploadId, check.Matches, ".+") - c.Assert(multis[1].Bucket, check.Equals, b) - c.Assert(multis[1].Key, check.Equals, "a/multi3") - c.Assert(multis[1].UploadId, check.Matches, ".+") -} - -func (s *ClientTests) TestMultiPutAllZeroLength(c *check.C) { - b := testBucket(s.s3) - err := b.PutBucket(s3.Private) - c.Assert(err, check.IsNil) - - multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{}) - c.Assert(err, check.IsNil) - defer multi.Abort() - - // This tests an edge case. Amazon requires at least one - // part for multiprat uploads to work, even the part is empty. - parts, err := multi.PutAll(strings.NewReader(""), 5*1024*1024) - c.Assert(err, check.IsNil) - c.Assert(parts, check.HasLen, 1) - c.Assert(parts[0].Size, check.Equals, int64(0)) - c.Assert(parts[0].ETag, check.Equals, `"d41d8cd98f00b204e9800998ecf8427e"`) - - err = multi.Complete(parts) - c.Assert(err, check.IsNil) -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3t_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3t_test.go deleted file mode 100644 index 72279ff3..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3t_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package s3_test - -import ( - "github.com/AdRoll/goamz/aws" - "github.com/AdRoll/goamz/s3" - "github.com/AdRoll/goamz/s3/s3test" - "github.com/AdRoll/goamz/testutil" - "gopkg.in/check.v1" -) - -type LocalServer struct { - auth aws.Auth - region aws.Region - srv *s3test.Server - config *s3test.Config -} - -func (s *LocalServer) SetUp(c *check.C) { - srv, err := s3test.NewServer(s.config) - c.Assert(err, check.IsNil) - c.Assert(srv, check.NotNil) - - s.srv = srv - s.region = aws.Region{ - Name: "faux-region-1", - S3Endpoint: srv.URL(), - S3LocationConstraint: true, // s3test server requires a LocationConstraint - } -} - -// LocalServerSuite defines tests that will run -// against the local s3test server. It includes -// selected tests from ClientTests; -// when the s3test functionality is sufficient, it should -// include all of them, and ClientTests can be simply embedded. -type LocalServerSuite struct { - srv LocalServer - clientTests ClientTests -} - -var ( - // run tests twice, once in us-east-1 mode, once not. - _ = check.Suite(&LocalServerSuite{}) - _ = check.Suite(&LocalServerSuite{ - srv: LocalServer{ - config: &s3test.Config{ - Send409Conflict: true, - }, - }, - }) -) - -func (s *LocalServerSuite) SetUpSuite(c *check.C) { - s.srv.SetUp(c) - s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region) - - // TODO Sadly the fake server ignores auth completely right now. :-( - s.clientTests.authIsBroken = true - s.clientTests.Cleanup() -} - -func (s *LocalServerSuite) TearDownTest(c *check.C) { - s.clientTests.Cleanup() -} - -func (s *LocalServerSuite) TestBasicFunctionality(c *check.C) { - s.clientTests.TestBasicFunctionality(c) -} - -func (s *LocalServerSuite) TestGetNotFound(c *check.C) { - s.clientTests.TestGetNotFound(c) -} - -func (s *LocalServerSuite) TestBucketList(c *check.C) { - s.clientTests.TestBucketList(c) -} - -func (s *LocalServerSuite) TestDoublePutBucket(c *check.C) { - s.clientTests.TestDoublePutBucket(c) -} - -func (s *LocalServerSuite) TestMultiComplete(c *check.C) { - if !testutil.Amazon { - c.Skip("live tests against AWS disabled (no -amazon)") - } - s.clientTests.TestMultiComplete(c) -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/sign_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/sign_test.go deleted file mode 100644 index 613dc766..00000000 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/sign_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package s3_test - -import ( - "github.com/AdRoll/goamz/aws" - "github.com/AdRoll/goamz/s3" - "gopkg.in/check.v1" -) - -// S3 ReST authentication docs: http://goo.gl/G1LrK - -var testAuth = aws.Auth{AccessKey: "0PN5J17HBGZHT7JJ3X82", SecretKey: "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"} - -func (s *S) TestSignExampleObjectGet(c *check.C) { - method := "GET" - path := "/johnsmith/photos/puppy.jpg" - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:36:42 +0000"}, - } - s3.Sign(testAuth, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleObjectPut(c *check.C) { - method := "PUT" - path := "/johnsmith/photos/puppy.jpg" - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 21:15:45 +0000"}, - "Content-Type": {"image/jpeg"}, - "Content-Length": {"94328"}, - } - s3.Sign(testAuth, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleList(c *check.C) { - method := "GET" - path := "/johnsmith/" - params := map[string][]string{ - "prefix": {"photos"}, - "max-keys": {"50"}, - "marker": {"puppy"}, - } - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:42:41 +0000"}, - "User-Agent": {"Mozilla/5.0"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleFetch(c *check.C) { - method := "GET" - path := "/johnsmith/" - params := map[string][]string{ - "acl": {""}, - } - headers := map[string][]string{ - "Host": {"johnsmith.s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 19:44:46 +0000"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleDelete(c *check.C) { - method := "DELETE" - path := "/johnsmith/photos/puppy.jpg" - params := map[string][]string{} - headers := map[string][]string{ - "Host": {"s3.amazonaws.com"}, - "Date": {"Tue, 27 Mar 2007 21:20:27 +0000"}, - "User-Agent": {"dotnet"}, - "x-amz-date": {"Tue, 27 Mar 2007 21:20:26 +0000"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleUpload(c *check.C) { - method := "PUT" - path := "/static.johnsmith.net/db-backup.dat.gz" - params := map[string][]string{} - headers := map[string][]string{ - "Host": {"static.johnsmith.net:8080"}, - "Date": {"Tue, 27 Mar 2007 21:06:08 +0000"}, - "User-Agent": {"curl/7.15.5"}, - "x-amz-acl": {"public-read"}, - "content-type": {"application/x-download"}, - "Content-MD5": {"4gJE4saaMU4BqNR0kLY+lw=="}, - "X-Amz-Meta-ReviewedBy": {"joe@johnsmith.net,jane@johnsmith.net"}, - "X-Amz-Meta-FileChecksum": {"0x02661779"}, - "X-Amz-Meta-ChecksumAlgorithm": {"crc32"}, - "Content-Disposition": {"attachment; filename=database.dat"}, - "Content-Encoding": {"gzip"}, - "Content-Length": {"5913339"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleListAllMyBuckets(c *check.C) { - method := "GET" - path := "/" - headers := map[string][]string{ - "Host": {"s3.amazonaws.com"}, - "Date": {"Wed, 28 Mar 2007 01:29:59 +0000"}, - } - s3.Sign(testAuth, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:Db+gepJSUbZKwpx1FR0DLtEYoZA=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleUnicodeKeys(c *check.C) { - method := "GET" - path := "/dictionary/fran%C3%A7ais/pr%c3%a9f%c3%a8re" - headers := map[string][]string{ - "Host": {"s3.amazonaws.com"}, - "Date": {"Wed, 28 Mar 2007 01:49:49 +0000"}, - } - s3.Sign(testAuth, method, path, nil, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:dxhSBHoI6eVSPcXJqEghlUzZMnY=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} - -func (s *S) TestSignExampleCustomSSE(c *check.C) { - method := "GET" - path := "/secret/config" - params := map[string][]string{} - headers := map[string][]string{ - "Host": {"secret.johnsmith.net:8080"}, - "Date": {"Tue, 27 Mar 2007 21:06:08 +0000"}, - "x-amz-server-side-encryption-customer-key": {"MWJhakVna1dQT1B0SDFMeGtVVnRQRTFGaU1ldFJrU0I="}, - "x-amz-server-side-encryption-customer-key-MD5": {"glIqxpqQ4a9aoK/iLttKzQ=="}, - "x-amz-server-side-encryption-customer-algorithm": {"AES256"}, - } - s3.Sign(testAuth, method, path, params, headers) - expected := "AWS 0PN5J17HBGZHT7JJ3X82:Xq6PWmIo0aOWq+LDjCEiCGgbmHE=" - c.Assert(headers["Authorization"], check.DeepEquals, []string{expected}) -} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/attempt.go b/Godeps/_workspace/src/github.com/docker/goamz/aws/attempt.go similarity index 100% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/aws/attempt.go rename to Godeps/_workspace/src/github.com/docker/goamz/aws/attempt.go diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go b/Godeps/_workspace/src/github.com/docker/goamz/aws/aws.go similarity index 100% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go rename to Godeps/_workspace/src/github.com/docker/goamz/aws/aws.go diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/client.go b/Godeps/_workspace/src/github.com/docker/goamz/aws/client.go similarity index 100% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/aws/client.go rename to Godeps/_workspace/src/github.com/docker/goamz/aws/client.go diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go b/Godeps/_workspace/src/github.com/docker/goamz/aws/regions.go similarity index 100% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go rename to Godeps/_workspace/src/github.com/docker/goamz/aws/regions.go diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/retry.go b/Godeps/_workspace/src/github.com/docker/goamz/aws/retry.go similarity index 100% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/aws/retry.go rename to Godeps/_workspace/src/github.com/docker/goamz/aws/retry.go diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go b/Godeps/_workspace/src/github.com/docker/goamz/aws/sign.go similarity index 93% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go rename to Godeps/_workspace/src/github.com/docker/goamz/aws/sign.go index ea296013..d1640d44 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go +++ b/Godeps/_workspace/src/github.com/docker/goamz/aws/sign.go @@ -307,20 +307,42 @@ func (s *V4Signer) canonicalURI(u *url.URL) string { } func (s *V4Signer) canonicalQueryString(u *url.URL) string { - var a []string + keyValues := make(map[string]string, len(u.Query())) + keys := make([]string, len(u.Query())) + + key_i := 0 for k, vs := range u.Query() { k = url.QueryEscape(k) - for _, v := range vs { - if v == "" { - a = append(a, k+"=") - } else { - v = url.QueryEscape(v) - a = append(a, k+"="+v) - } + + a := make([]string, len(vs)) + for idx, v := range vs { + v = url.QueryEscape(v) + a[idx] = fmt.Sprintf("%s=%s", k, v) } + + keyValues[k] = strings.Join(a, "&") + keys[key_i] = k + key_i++ } - sort.Strings(a) - return strings.Join(a, "&") + + sort.Strings(keys) + + query := make([]string, len(keys)) + for idx, key := range keys { + query[idx] = keyValues[key] + } + + query_str := strings.Join(query, "&") + + // AWS V4 signing requires that the space characters + // are encoded as %20 instead of +. On the other hand + // golangs url.QueryEscape as well as url.Values.Encode() + // both encode the space as a + character. See: + // http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + // https://github.com/golang/go/issues/4013 + // https://groups.google.com/forum/#!topic/golang-nuts/BB443qEjPIk + + return strings.Replace(query_str, "+", "%20", -1) } func (s *V4Signer) canonicalHeaders(h http.Header) string { diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/cloudfront.go b/Godeps/_workspace/src/github.com/docker/goamz/cloudfront/cloudfront.go similarity index 99% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/cloudfront.go rename to Godeps/_workspace/src/github.com/docker/goamz/cloudfront/cloudfront.go index b845d3c5..0bffe88a 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/cloudfront/cloudfront.go +++ b/Godeps/_workspace/src/github.com/docker/goamz/cloudfront/cloudfront.go @@ -7,7 +7,7 @@ import ( "encoding/base64" "encoding/json" "fmt" - "github.com/AdRoll/goamz/aws" + "github.com/docker/goamz/aws" "net/url" "strconv" "strings" diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/lifecycle.go b/Godeps/_workspace/src/github.com/docker/goamz/s3/lifecycle.go similarity index 100% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/s3/lifecycle.go rename to Godeps/_workspace/src/github.com/docker/goamz/s3/lifecycle.go diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi.go b/Godeps/_workspace/src/github.com/docker/goamz/s3/multi.go similarity index 100% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi.go rename to Godeps/_workspace/src/github.com/docker/goamz/s3/multi.go diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go b/Godeps/_workspace/src/github.com/docker/goamz/s3/s3.go similarity index 96% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go rename to Godeps/_workspace/src/github.com/docker/goamz/s3/s3.go index c8d4a570..5dedaf10 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go +++ b/Godeps/_workspace/src/github.com/docker/goamz/s3/s3.go @@ -30,7 +30,7 @@ import ( "strings" "time" - "github.com/AdRoll/goamz/aws" + "github.com/docker/goamz/aws" ) const debug = false @@ -39,10 +39,9 @@ const debug = false type S3 struct { aws.Auth aws.Region - ConnectTimeout time.Duration - ReadTimeout time.Duration - Signature int - private byte // Reserve the right of using private data. + Signature int + Client *http.Client + private byte // Reserve the right of using private data. } // The Bucket type encapsulates operations with an S3 bucket. @@ -61,6 +60,8 @@ type Owner struct { // type Options struct { SSE bool + SSEKMS bool + SSEKMSKeyId string SSECustomerAlgorithm string SSECustomerKey string SSECustomerKeyMD5 string @@ -96,7 +97,13 @@ var attempts = aws.AttemptStrategy{ // New creates a new S3. func New(auth aws.Auth, region aws.Region) *S3 { - return &S3{auth, region, 0, 0, aws.V2Signature, 0} + return &S3{ + Auth: auth, + Region: region, + Signature: aws.V2Signature, + Client: http.DefaultClient, + private: 0, + } } // Bucket returns a Bucket with the given name. @@ -171,6 +178,13 @@ const ( StandardStorage = StorageClass("STANDARD") ) +type ServerSideEncryption string + +const ( + S3Managed = ServerSideEncryption("AES256") + KMSManaged = ServerSideEncryption("aws:kms") +) + // PutBucket creates a new bucket. // // See http://goo.gl/ndjnR for details. @@ -344,7 +358,7 @@ func (b *Bucket) Put(path string, data []byte, contType string, perm ACL, option func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (*CopyObjectResult, error) { headers := map[string][]string{ "x-amz-acl": {string(perm)}, - "x-amz-copy-source": {url.QueryEscape(source)}, + "x-amz-copy-source": {escapePath(source)}, } options.addHeaders(headers) req := &request{ @@ -383,7 +397,12 @@ func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType stri // addHeaders adds o's specified fields to headers func (o Options) addHeaders(headers map[string][]string) { if o.SSE { - headers["x-amz-server-side-encryption"] = []string{"AES256"} + headers["x-amz-server-side-encryption"] = []string{string(S3Managed)} + } else if o.SSEKMS { + headers["x-amz-server-side-encryption"] = []string{string(KMSManaged)} + if len(o.SSEKMSKeyId) != 0 { + headers["x-amz-server-side-encryption-aws-kms-key-id"] = []string{o.SSEKMSKeyId} + } } else if len(o.SSECustomerAlgorithm) != 0 && len(o.SSECustomerKey) != 0 && len(o.SSECustomerKeyMD5) != 0 { // Amazon-managed keys and customer-managed keys are mutually exclusive headers["x-amz-server-side-encryption-customer-algorithm"] = []string{o.SSECustomerAlgorithm} @@ -886,6 +905,12 @@ func (b *Bucket) PostFormArgsEx(path string, expires time.Time, redirect string, "key": path, } + if token := b.S3.Auth.Token(); token != "" { + fields["x-amz-security-token"] = token + conditions = append(conditions, + fmt.Sprintf("{\"x-amz-security-token\": \"%s\"}", token)) + } + if conds != nil { conditions = append(conditions, conds...) } @@ -1123,7 +1148,6 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) { Method: req.method, ProtoMajor: 1, ProtoMinor: 1, - Close: true, Header: req.headers, Form: req.params, } @@ -1143,28 +1167,7 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) { // If resp is not nil, the XML data contained in the response // body will be unmarshalled on it. func (s3 *S3) doHttpRequest(hreq *http.Request, resp interface{}) (*http.Response, error) { - c := http.Client{ - Transport: &http.Transport{ - Dial: func(netw, addr string) (c net.Conn, err error) { - deadline := time.Now().Add(s3.ReadTimeout) - if s3.ConnectTimeout > 0 { - c, err = net.DialTimeout(netw, addr, s3.ConnectTimeout) - } else { - c, err = net.Dial(netw, addr) - } - if err != nil { - return - } - if s3.ReadTimeout > 0 { - err = c.SetDeadline(deadline) - } - return - }, - Proxy: http.ProxyFromEnvironment, - }, - } - - hresp, err := c.Do(hreq) + hresp, err := s3.Client.Do(hreq) if err != nil { return nil, err } @@ -1296,3 +1299,7 @@ func hasCode(err error, code string) bool { s3err, ok := err.(*Error) return ok && s3err.Code == code } + +func escapePath(s string) string { + return (&url.URL{Path: s}).String() +} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go b/Godeps/_workspace/src/github.com/docker/goamz/s3/s3test/server.go similarity index 94% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go rename to Godeps/_workspace/src/github.com/docker/goamz/s3/s3test/server.go index 2e4e3f24..ed3eb988 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go +++ b/Godeps/_workspace/src/github.com/docker/goamz/s3/s3test/server.go @@ -7,7 +7,7 @@ import ( "encoding/hex" "encoding/xml" "fmt" - "github.com/AdRoll/goamz/s3" + "github.com/docker/goamz/s3" "io" "io/ioutil" "log" @@ -44,6 +44,17 @@ type action struct { reqId string } +// A Clock reports the current time. +type Clock interface { + Now() time.Time +} + +type realClock struct{} + +func (c *realClock) Now() time.Time { + return time.Now() +} + // Config controls the internal behaviour of the Server. A nil config is the default // and behaves as if all configurations assume their default behaviour. Once passed // to NewServer, the configuration must not be modified. @@ -58,6 +69,10 @@ type Config struct { // Address on which to listen. By default, a random port is assigned by the // operating system and the server listens on localhost. ListenAddress string + + // Clock used to set mtime when updating an object. If nil, + // use the real clock. + Clock Clock } func (c *Config) send409Conflict() bool { @@ -76,6 +91,7 @@ type Server struct { mu sync.Mutex buckets map[string]*bucket config *Config + closed bool } type bucket struct { @@ -129,10 +145,18 @@ type resource interface { func NewServer(config *Config) (*Server, error) { listenAddress := "localhost:0" - if config != nil && config.ListenAddress != "" { + if config == nil { + config = &Config{} + } + + if config.ListenAddress != "" { listenAddress = config.ListenAddress } + if config.Clock == nil { + config.Clock = &realClock{} + } + l, err := net.Listen("tcp", listenAddress) if err != nil { return nil, fmt.Errorf("cannot listen on localhost: %v", err) @@ -151,6 +175,10 @@ func NewServer(config *Config) (*Server, error) { // Quit closes down the server. func (srv *Server) Quit() { + srv.mu.Lock() + srv.closed = true + srv.mu.Unlock() + srv.listener.Close() } @@ -175,6 +203,13 @@ func (srv *Server) serveHTTP(w http.ResponseWriter, req *http.Request) { srv.mu.Lock() defer srv.mu.Unlock() + if srv.closed { + hj := w.(http.Hijacker) + conn, _, _ := hj.Hijack() + conn.Close() + return + } + if debug { log.Printf("s3test %q %q", req.Method, req.URL) } @@ -360,15 +395,28 @@ func (r bucketResource) get(a *action) interface{} { if maxKeys <= 0 { maxKeys = 1000 } - resp := &s3.ListResp{ - Name: r.bucket.name, - Prefix: prefix, - Delimiter: delimiter, - Marker: marker, - MaxKeys: maxKeys, + + type commonPrefix struct { + Prefix string } - var prefixes []string + type serverListResponse struct { + s3.ListResp + CommonPrefixes []commonPrefix + } + + resp := &serverListResponse{ + ListResp: s3.ListResp{ + Name: r.bucket.name, + Prefix: prefix, + Delimiter: delimiter, + Marker: marker, + MaxKeys: maxKeys, + }, + } + + var prefixes []commonPrefix + var lastName string for _, obj := range objs { if !strings.HasPrefix(obj.name, prefix) { continue @@ -378,7 +426,7 @@ func (r bucketResource) get(a *action) interface{} { if delimiter != "" { if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 { name = obj.name[:len(prefix)+i+len(delimiter)] - if prefixes != nil && prefixes[len(prefixes)-1] == name { + if prefixes != nil && prefixes[len(prefixes)-1].Prefix == name { continue } isPrefix = true @@ -389,14 +437,16 @@ func (r bucketResource) get(a *action) interface{} { } if len(resp.Contents)+len(prefixes) >= maxKeys { resp.IsTruncated = true + resp.NextMarker = lastName break } if isPrefix { - prefixes = append(prefixes, name) + prefixes = append(prefixes, commonPrefix{Prefix: name}) } else { // Contents contains only keys not found in CommonPrefixes resp.Contents = append(resp.Contents, obj.s3Key()) } + lastName = name } resp.CommonPrefixes = prefixes return resp @@ -649,7 +699,7 @@ func (objr objectResource) get(a *action) interface{} { // TODO x-amz-request-id h.Set("Content-Length", fmt.Sprint(len(data))) h.Set("ETag", "\""+hex.EncodeToString(obj.checksum)+"\"") - h.Set("Last-Modified", obj.mtime.Format(lastModifiedTimeFormat)) + h.Set("Last-Modified", obj.mtime.UTC().Format(lastModifiedTimeFormat)) if status != http.StatusOK { a.w.WriteHeader(status) @@ -682,6 +732,8 @@ func (objr objectResource) put(a *action) interface{} { // TODO x-amz-server-side-encryption // TODO x-amz-storage-class + var res interface{} + uploadId := a.req.URL.Query().Get("uploadId") var partNumber uint @@ -751,6 +803,7 @@ func (objr objectResource) put(a *action) interface{} { obj.meta[key] = values } } + obj.mtime = a.srv.config.Clock.Now() if copySource := a.req.Header.Get("X-Amz-Copy-Source"); copySource != "" { idx := strings.IndexByte(copySource, '/') @@ -786,11 +839,15 @@ func (objr objectResource) put(a *action) interface{} { obj.meta[k] = make([]string, len(v)) copy(obj.meta[k], v) } + + res = &s3.CopyObjectResult{ + ETag: etag, + LastModified: obj.mtime.UTC().Format(time.RFC3339), + } } else { obj.data = data obj.checksum = gotHash } - obj.mtime = time.Now() objr.bucket.objects[objr.name] = obj } else { // For multipart commit @@ -800,13 +857,13 @@ func (objr objectResource) put(a *action) interface{} { index: partNumber, data: data, etag: etag, - lastModified: time.Now(), + lastModified: a.srv.config.Clock.Now(), } objr.bucket.multipartUploads[uploadId] = append(parts, part) } - return nil + return res } func (objr objectResource) delete(a *action) interface{} { diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/sign.go b/Godeps/_workspace/src/github.com/docker/goamz/s3/sign.go similarity index 98% rename from Godeps/_workspace/src/github.com/AdRoll/goamz/s3/sign.go rename to Godeps/_workspace/src/github.com/docker/goamz/s3/sign.go index 19642094..6f24c667 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/sign.go +++ b/Godeps/_workspace/src/github.com/docker/goamz/s3/sign.go @@ -4,7 +4,7 @@ import ( "crypto/hmac" "crypto/sha1" "encoding/base64" - "github.com/AdRoll/goamz/aws" + "github.com/docker/goamz/aws" "log" "sort" "strings" diff --git a/registry/handlers/app.go b/registry/handlers/app.go index 23225493..6dabaca3 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "runtime" "time" log "github.com/Sirupsen/logrus" @@ -30,6 +31,7 @@ import ( storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/factory" storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware" + "github.com/docker/distribution/version" "github.com/docker/libtrust" "github.com/garyburd/redigo/redis" "github.com/gorilla/mux" @@ -83,12 +85,12 @@ type App struct { // NewApp takes a configuration and returns a configured app, ready to serve // requests. The app only implements ServeHTTP and can be wrapped in other // handlers accordingly. -func NewApp(ctx context.Context, configuration *configuration.Configuration) *App { +func NewApp(ctx context.Context, config *configuration.Configuration) *App { app := &App{ - Config: configuration, + Config: config, Context: ctx, - router: v2.RouterWithPrefix(configuration.HTTP.Prefix), - isCache: configuration.Proxy.RemoteURL != "", + router: v2.RouterWithPrefix(config.HTTP.Prefix), + isCache: config.Proxy.RemoteURL != "", } // Register the handler dispatchers. @@ -102,8 +104,15 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap app.register(v2.RouteNameBlobUpload, blobUploadDispatcher) app.register(v2.RouteNameBlobUploadChunk, blobUploadDispatcher) + // override the storage driver's UA string for registry outbound HTTP requests + storageParams := config.Storage.Parameters() + if storageParams == nil { + storageParams = make(configuration.Parameters) + } + storageParams["useragent"] = fmt.Sprintf("docker-distribution/%s %s", version.Version, runtime.Version()) + var err error - app.driver, err = factory.Create(configuration.Storage.Type(), configuration.Storage.Parameters()) + app.driver, err = factory.Create(config.Storage.Type(), storageParams) if err != nil { // TODO(stevvooe): Move the creation of a service into a protected // method, where this is created lazily. Its status can be queried via @@ -112,7 +121,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap } purgeConfig := uploadPurgeDefaultConfig() - if mc, ok := configuration.Storage["maintenance"]; ok { + if mc, ok := config.Storage["maintenance"]; ok { if v, ok := mc["uploadpurging"]; ok { purgeConfig, ok = v.(map[interface{}]interface{}) if !ok { @@ -135,15 +144,15 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap startUploadPurger(app, app.driver, ctxu.GetLogger(app), purgeConfig) - app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"]) + app.driver, err = applyStorageMiddleware(app.driver, config.Middleware["storage"]) if err != nil { panic(err) } - app.configureSecret(configuration) - app.configureEvents(configuration) - app.configureRedis(configuration) - app.configureLogHook(configuration) + app.configureSecret(config) + app.configureEvents(config) + app.configureRedis(config) + app.configureLogHook(config) // Generate an ephemeral key to be used for signing converted manifests // for clients that don't support schema2. @@ -152,8 +161,8 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap panic(err) } - if configuration.HTTP.Host != "" { - u, err := url.Parse(configuration.HTTP.Host) + if config.HTTP.Host != "" { + u, err := url.Parse(config.HTTP.Host) if err != nil { panic(fmt.Sprintf(`could not parse http "host" parameter: %v`, err)) } @@ -167,7 +176,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap } // configure deletion - if d, ok := configuration.Storage["delete"]; ok { + if d, ok := config.Storage["delete"]; ok { e, ok := d["enabled"] if ok { if deleteEnabled, ok := e.(bool); ok && deleteEnabled { @@ -178,7 +187,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap // configure redirects var redirectDisabled bool - if redirectConfig, ok := configuration.Storage["redirect"]; ok { + if redirectConfig, ok := config.Storage["redirect"]; ok { v := redirectConfig["disable"] switch v := v.(type) { case bool: @@ -194,7 +203,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap } // configure storage caches - if cc, ok := configuration.Storage["cache"]; ok { + if cc, ok := config.Storage["cache"]; ok { v, ok := cc["blobdescriptor"] if !ok { // Backwards compatible: "layerinfo" == "blobdescriptor" @@ -223,7 +232,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap ctxu.GetLogger(app).Infof("using inmemory blob descriptor cache") default: if v != "" { - ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", configuration.Storage["cache"]) + ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", config.Storage["cache"]) } } } @@ -236,15 +245,15 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap } } - app.registry, err = applyRegistryMiddleware(app.Context, app.registry, configuration.Middleware["registry"]) + app.registry, err = applyRegistryMiddleware(app.Context, app.registry, config.Middleware["registry"]) if err != nil { panic(err) } - authType := configuration.Auth.Type() + authType := config.Auth.Type() if authType != "" { - accessController, err := auth.GetAccessController(configuration.Auth.Type(), configuration.Auth.Parameters()) + accessController, err := auth.GetAccessController(config.Auth.Type(), config.Auth.Parameters()) if err != nil { panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err)) } @@ -253,13 +262,13 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap } // configure as a pull through cache - if configuration.Proxy.RemoteURL != "" { - app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, configuration.Proxy) + if config.Proxy.RemoteURL != "" { + app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, config.Proxy) if err != nil { panic(err.Error()) } app.isCache = true - ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", configuration.Proxy.RemoteURL) + ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", config.Proxy.RemoteURL) } return app diff --git a/registry/storage/driver/middleware/cloudfront/middleware.go b/registry/storage/driver/middleware/cloudfront/middleware.go index 31c00afc..56edda3a 100644 --- a/registry/storage/driver/middleware/cloudfront/middleware.go +++ b/registry/storage/driver/middleware/cloudfront/middleware.go @@ -10,10 +10,10 @@ import ( "io/ioutil" "time" - "github.com/AdRoll/goamz/cloudfront" "github.com/docker/distribution/context" storagedriver "github.com/docker/distribution/registry/storage/driver" storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware" + "github.com/docker/goamz/cloudfront" ) // cloudFrontStorageMiddleware provides an simple implementation of layerHandler that diff --git a/registry/storage/driver/s3/s3.go b/registry/storage/driver/s3/s3.go index 7bb23a85..f09e5508 100644 --- a/registry/storage/driver/s3/s3.go +++ b/registry/storage/driver/s3/s3.go @@ -26,11 +26,12 @@ import ( "sync" "time" - "github.com/AdRoll/goamz/aws" - "github.com/AdRoll/goamz/s3" "github.com/Sirupsen/logrus" + "github.com/docker/goamz/aws" + "github.com/docker/goamz/s3" "github.com/docker/distribution/context" + "github.com/docker/distribution/registry/client/transport" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/base" "github.com/docker/distribution/registry/storage/driver/factory" @@ -58,6 +59,7 @@ type DriverParameters struct { V4Auth bool ChunkSize int64 RootDirectory string + UserAgent string } func init() { @@ -168,7 +170,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { case int, uint, int32, uint32, uint64: chunkSize = reflect.ValueOf(v).Convert(reflect.TypeOf(chunkSize)).Int() default: - return nil, fmt.Errorf("invalid valud for chunksize: %#v", chunkSizeParam) + return nil, fmt.Errorf("invalid value for chunksize: %#v", chunkSizeParam) } if chunkSize < minChunkSize { @@ -181,6 +183,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { rootDirectory = "" } + userAgent, ok := parameters["useragent"] + if !ok { + userAgent = "" + } + params := DriverParameters{ fmt.Sprint(accessKey), fmt.Sprint(secretKey), @@ -191,6 +198,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { v4AuthBool, chunkSize, fmt.Sprint(rootDirectory), + fmt.Sprint(userAgent), } return New(params) @@ -209,7 +217,16 @@ func New(params DriverParameters) (*Driver, error) { } s3obj := s3.New(auth, params.Region) - bucket := s3obj.Bucket(params.Bucket) + + if params.UserAgent != "" { + s3obj.Client = &http.Client{ + Transport: transport.NewTransport(http.DefaultTransport, + transport.NewHeaderRequestModifier(http.Header{ + http.CanonicalHeaderKey("User-Agent"): []string{params.UserAgent}, + }), + ), + } + } if params.V4Auth { s3obj.Signature = aws.V4Signature @@ -219,6 +236,8 @@ func New(params DriverParameters) (*Driver, error) { } } + bucket := s3obj.Bucket(params.Bucket) + // TODO Currently multipart uploads have no timestamps, so this would be unwise // if you initiated a new s3driver while another one is running on the same bucket. // multis, _, err := bucket.ListMulti("", "") diff --git a/registry/storage/driver/s3/s3_test.go b/registry/storage/driver/s3/s3_test.go index 70172a6d..86f433f3 100644 --- a/registry/storage/driver/s3/s3_test.go +++ b/registry/storage/driver/s3/s3_test.go @@ -6,10 +6,10 @@ import ( "strconv" "testing" - "github.com/AdRoll/goamz/aws" "github.com/docker/distribution/context" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/testsuites" + "github.com/docker/goamz/aws" "gopkg.in/check.v1" ) @@ -69,6 +69,7 @@ func init() { v4AuthBool, minChunkSize, rootDirectory, + "", } return New(parameters)