Merge pull request #1381 from BrianBland/s3CustomUAString

Adds custom registry User-Agent header to s3 HTTP requests
This commit is contained in:
Aaron Lehmann 2016-02-01 15:40:36 -08:00
commit 2fc586d2a4
32 changed files with 214 additions and 3566 deletions

24
Godeps/Godeps.json generated
View file

@ -41,18 +41,6 @@
"ImportPath": "google.golang.org/cloud/storage",
"Rev": "2400193c85c3561d13880d34e0e10c4315bb02af"
},
{
"ImportPath": "github.com/AdRoll/goamz/aws",
"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
},
{
"ImportPath": "github.com/AdRoll/goamz/cloudfront",
"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
},
{
"ImportPath": "github.com/AdRoll/goamz/s3",
"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
},
{
"ImportPath": "github.com/Azure/azure-sdk-for-go/storage",
"Rev": "97d9593768bbbbd316f9c055dfc5f780933cd7fc"
@ -87,6 +75,18 @@
"ImportPath": "github.com/denverdino/aliyungo/common",
"Rev": "6ffb587da9da6d029d0ce517b85fecc82172d502"
},
{
"ImportPath": "github.com/docker/goamz/aws",
"Rev": "fb9c4c25c583d56a0544da8d1094294908c68ee8"
},
{
"ImportPath": "github.com/docker/goamz/cloudfront",
"Rev": "fb9c4c25c583d56a0544da8d1094294908c68ee8"
},
{
"ImportPath": "github.com/docker/goamz/s3",
"Rev": "fb9c4c25c583d56a0544da8d1094294908c68ee8"
},
{
"ImportPath": "github.com/docker/libtrust",
"Rev": "fa567046d9b14f6aa788882a950d69651d230b21"

View file

@ -1,57 +0,0 @@
package aws_test
import (
"github.com/AdRoll/goamz/aws"
"gopkg.in/check.v1"
"time"
)
func (S) TestAttemptTiming(c *check.C) {
testAttempt := aws.AttemptStrategy{
Total: 0.25e9,
Delay: 0.1e9,
}
want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9}
got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing
t0 := time.Now()
for a := testAttempt.Start(); a.Next(); {
got = append(got, time.Now().Sub(t0))
}
got = append(got, time.Now().Sub(t0))
c.Assert(got, check.HasLen, len(want))
const margin = 0.01e9
for i, got := range want {
lo := want[i] - margin
hi := want[i] + margin
if got < lo || got > hi {
c.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds())
}
}
}
func (S) TestAttemptNextHasNext(c *check.C) {
a := aws.AttemptStrategy{}.Start()
c.Assert(a.Next(), check.Equals, true)
c.Assert(a.Next(), check.Equals, false)
a = aws.AttemptStrategy{}.Start()
c.Assert(a.Next(), check.Equals, true)
c.Assert(a.HasNext(), check.Equals, false)
c.Assert(a.Next(), check.Equals, false)
a = aws.AttemptStrategy{Total: 2e8}.Start()
c.Assert(a.Next(), check.Equals, true)
c.Assert(a.HasNext(), check.Equals, true)
time.Sleep(2e8)
c.Assert(a.HasNext(), check.Equals, true)
c.Assert(a.Next(), check.Equals, true)
c.Assert(a.Next(), check.Equals, false)
a = aws.AttemptStrategy{Total: 1e8, Min: 2}.Start()
time.Sleep(1e8)
c.Assert(a.Next(), check.Equals, true)
c.Assert(a.HasNext(), check.Equals, true)
c.Assert(a.Next(), check.Equals, true)
c.Assert(a.HasNext(), check.Equals, false)
c.Assert(a.Next(), check.Equals, false)
}

View file

@ -1,140 +0,0 @@
package aws_test
import (
"github.com/AdRoll/goamz/aws"
"gopkg.in/check.v1"
"io/ioutil"
"os"
"strings"
"testing"
"time"
)
func Test(t *testing.T) {
check.TestingT(t)
}
var _ = check.Suite(&S{})
type S struct {
environ []string
}
func (s *S) SetUpSuite(c *check.C) {
s.environ = os.Environ()
}
func (s *S) TearDownTest(c *check.C) {
os.Clearenv()
for _, kv := range s.environ {
l := strings.SplitN(kv, "=", 2)
os.Setenv(l[0], l[1])
}
}
func (s *S) TestEnvAuthNoSecret(c *check.C) {
os.Clearenv()
_, err := aws.EnvAuth()
c.Assert(err, check.ErrorMatches, "AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment")
}
func (s *S) TestEnvAuthNoAccess(c *check.C) {
os.Clearenv()
os.Setenv("AWS_SECRET_ACCESS_KEY", "foo")
_, err := aws.EnvAuth()
c.Assert(err, check.ErrorMatches, "AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment")
}
func (s *S) TestEnvAuth(c *check.C) {
os.Clearenv()
os.Setenv("AWS_SECRET_ACCESS_KEY", "secret")
os.Setenv("AWS_ACCESS_KEY_ID", "access")
auth, err := aws.EnvAuth()
c.Assert(err, check.IsNil)
c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
}
func (s *S) TestEnvAuthAlt(c *check.C) {
os.Clearenv()
os.Setenv("AWS_SECRET_KEY", "secret")
os.Setenv("AWS_ACCESS_KEY", "access")
auth, err := aws.EnvAuth()
c.Assert(err, check.IsNil)
c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
}
func (s *S) TestGetAuthStatic(c *check.C) {
exptdate := time.Now().Add(time.Hour)
auth, err := aws.GetAuth("access", "secret", "token", exptdate)
c.Assert(err, check.IsNil)
c.Assert(auth.AccessKey, check.Equals, "access")
c.Assert(auth.SecretKey, check.Equals, "secret")
c.Assert(auth.Token(), check.Equals, "token")
c.Assert(auth.Expiration(), check.Equals, exptdate)
}
func (s *S) TestGetAuthEnv(c *check.C) {
os.Clearenv()
os.Setenv("AWS_SECRET_ACCESS_KEY", "secret")
os.Setenv("AWS_ACCESS_KEY_ID", "access")
auth, err := aws.GetAuth("", "", "", time.Time{})
c.Assert(err, check.IsNil)
c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
}
func (s *S) TestEncode(c *check.C) {
c.Assert(aws.Encode("foo"), check.Equals, "foo")
c.Assert(aws.Encode("/"), check.Equals, "%2F")
}
func (s *S) TestRegionsAreNamed(c *check.C) {
for n, r := range aws.Regions {
c.Assert(n, check.Equals, r.Name)
}
}
func (s *S) TestCredentialsFileAuth(c *check.C) {
file, err := ioutil.TempFile("", "creds")
if err != nil {
c.Fatal(err)
}
iniFile := `
[default] ; comment 123
aws_access_key_id = keyid1 ;comment
aws_secret_access_key=key1
[profile2]
aws_access_key_id = keyid2 ;comment
aws_secret_access_key=key2
aws_session_token=token1
`
_, err = file.WriteString(iniFile)
if err != nil {
c.Fatal(err)
}
err = file.Close()
if err != nil {
c.Fatal(err)
}
// check non-existant profile
_, err = aws.CredentialFileAuth(file.Name(), "no profile", 30*time.Minute)
c.Assert(err, check.Not(check.Equals), nil)
defaultProfile, err := aws.CredentialFileAuth(file.Name(), "default", 30*time.Minute)
c.Assert(err, check.Equals, nil)
c.Assert(defaultProfile.AccessKey, check.Equals, "keyid1")
c.Assert(defaultProfile.SecretKey, check.Equals, "key1")
c.Assert(defaultProfile.Token(), check.Equals, "")
profile2, err := aws.CredentialFileAuth(file.Name(), "profile2", 30*time.Minute)
c.Assert(err, check.Equals, nil)
c.Assert(profile2.AccessKey, check.Equals, "keyid2")
c.Assert(profile2.SecretKey, check.Equals, "key2")
c.Assert(profile2.Token(), check.Equals, "token1")
}

View file

@ -1,29 +0,0 @@
package aws
import (
"net/http"
"time"
)
// V4Signer:
// Exporting methods for testing
func (s *V4Signer) RequestTime(req *http.Request) time.Time {
return s.requestTime(req)
}
func (s *V4Signer) CanonicalRequest(req *http.Request) string {
return s.canonicalRequest(req, "")
}
func (s *V4Signer) StringToSign(t time.Time, creq string) string {
return s.stringToSign(t, creq)
}
func (s *V4Signer) Signature(t time.Time, sts string) string {
return s.signature(t, sts)
}
func (s *V4Signer) Authorization(header http.Header, t time.Time, signature string) string {
return s.authorization(header, t, signature)
}

View file

@ -1,303 +0,0 @@
package aws
import (
"math/rand"
"net"
"net/http"
"testing"
"time"
)
type testInput struct {
res *http.Response
err error
numRetries int
}
type testResult struct {
shouldRetry bool
delay time.Duration
}
type testCase struct {
input testInput
defaultResult testResult
dynamoDBResult testResult
}
var testCases = []testCase{
// Test nil fields
testCase{
input: testInput{
err: nil,
res: nil,
numRetries: 0,
},
defaultResult: testResult{
shouldRetry: false,
delay: 300 * time.Millisecond,
},
dynamoDBResult: testResult{
shouldRetry: false,
delay: 25 * time.Millisecond,
},
},
// Test 3 different throttling exceptions
testCase{
input: testInput{
err: &Error{
Code: "Throttling",
},
numRetries: 0,
},
defaultResult: testResult{
shouldRetry: true,
delay: 617165505 * time.Nanosecond, // account for randomness with known seed
},
dynamoDBResult: testResult{
shouldRetry: true,
delay: 25 * time.Millisecond,
},
},
testCase{
input: testInput{
err: &Error{
Code: "ThrottlingException",
},
numRetries: 0,
},
defaultResult: testResult{
shouldRetry: true,
delay: 579393152 * time.Nanosecond, // account for randomness with known seed
},
dynamoDBResult: testResult{
shouldRetry: true,
delay: 25 * time.Millisecond,
},
},
testCase{
input: testInput{
err: &Error{
Code: "ProvisionedThroughputExceededException",
},
numRetries: 1,
},
defaultResult: testResult{
shouldRetry: true,
delay: 1105991654 * time.Nanosecond, // account for randomness with known seed
},
dynamoDBResult: testResult{
shouldRetry: true,
delay: 50 * time.Millisecond,
},
},
// Test a fake throttling exception
testCase{
input: testInput{
err: &Error{
Code: "MyMadeUpThrottlingCode",
},
numRetries: 0,
},
defaultResult: testResult{
shouldRetry: false,
delay: 300 * time.Millisecond,
},
dynamoDBResult: testResult{
shouldRetry: false,
delay: 25 * time.Millisecond,
},
},
// Test 5xx errors
testCase{
input: testInput{
res: &http.Response{
StatusCode: http.StatusInternalServerError,
},
numRetries: 1,
},
defaultResult: testResult{
shouldRetry: true,
delay: 600 * time.Millisecond,
},
dynamoDBResult: testResult{
shouldRetry: true,
delay: 50 * time.Millisecond,
},
},
testCase{
input: testInput{
res: &http.Response{
StatusCode: http.StatusServiceUnavailable,
},
numRetries: 1,
},
defaultResult: testResult{
shouldRetry: true,
delay: 600 * time.Millisecond,
},
dynamoDBResult: testResult{
shouldRetry: true,
delay: 50 * time.Millisecond,
},
},
// Test a random 400 error
testCase{
input: testInput{
res: &http.Response{
StatusCode: http.StatusNotFound,
},
numRetries: 1,
},
defaultResult: testResult{
shouldRetry: false,
delay: 600 * time.Millisecond,
},
dynamoDBResult: testResult{
shouldRetry: false,
delay: 50 * time.Millisecond,
},
},
// Test a temporary net.Error
testCase{
input: testInput{
res: &http.Response{},
err: &net.DNSError{
IsTimeout: true,
},
numRetries: 2,
},
defaultResult: testResult{
shouldRetry: true,
delay: 1200 * time.Millisecond,
},
dynamoDBResult: testResult{
shouldRetry: true,
delay: 100 * time.Millisecond,
},
},
// Test a non-temporary net.Error
testCase{
input: testInput{
res: &http.Response{},
err: &net.DNSError{
IsTimeout: false,
},
numRetries: 3,
},
defaultResult: testResult{
shouldRetry: false,
delay: 2400 * time.Millisecond,
},
dynamoDBResult: testResult{
shouldRetry: false,
delay: 200 * time.Millisecond,
},
},
// Assert failure after hitting max default retries
testCase{
input: testInput{
err: &Error{
Code: "ProvisionedThroughputExceededException",
},
numRetries: defaultMaxRetries,
},
defaultResult: testResult{
shouldRetry: false,
delay: 4313582352 * time.Nanosecond, // account for randomness with known seed
},
dynamoDBResult: testResult{
shouldRetry: true,
delay: 200 * time.Millisecond,
},
},
// Assert failure after hitting max DynamoDB retries
testCase{
input: testInput{
err: &Error{
Code: "ProvisionedThroughputExceededException",
},
numRetries: dynamoDBMaxRetries,
},
defaultResult: testResult{
shouldRetry: false,
delay: maxDelay,
},
dynamoDBResult: testResult{
shouldRetry: false,
delay: maxDelay,
},
},
// Assert we never go over the maxDelay value
testCase{
input: testInput{
numRetries: 25,
},
defaultResult: testResult{
shouldRetry: false,
delay: maxDelay,
},
dynamoDBResult: testResult{
shouldRetry: false,
delay: maxDelay,
},
},
}
func TestDefaultRetryPolicy(t *testing.T) {
rand.Seed(0)
var policy RetryPolicy
policy = &DefaultRetryPolicy{}
for _, test := range testCases {
res := test.input.res
err := test.input.err
numRetries := test.input.numRetries
shouldRetry := policy.ShouldRetry("", res, err, numRetries)
if shouldRetry != test.defaultResult.shouldRetry {
t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, test.defaultResult.shouldRetry, res, err, numRetries)
}
delay := policy.Delay("", res, err, numRetries)
if delay != test.defaultResult.delay {
t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, test.defaultResult.delay, res, err, numRetries)
}
}
}
func TestDynamoDBRetryPolicy(t *testing.T) {
var policy RetryPolicy
policy = &DynamoDBRetryPolicy{}
for _, test := range testCases {
res := test.input.res
err := test.input.err
numRetries := test.input.numRetries
shouldRetry := policy.ShouldRetry("", res, err, numRetries)
if shouldRetry != test.dynamoDBResult.shouldRetry {
t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, test.dynamoDBResult.shouldRetry, res, err, numRetries)
}
delay := policy.Delay("", res, err, numRetries)
if delay != test.dynamoDBResult.delay {
t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, test.dynamoDBResult.delay, res, err, numRetries)
}
}
}
func TestNeverRetryPolicy(t *testing.T) {
var policy RetryPolicy
policy = &NeverRetryPolicy{}
for _, test := range testCases {
res := test.input.res
err := test.input.err
numRetries := test.input.numRetries
shouldRetry := policy.ShouldRetry("", res, err, numRetries)
if shouldRetry {
t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, false, res, err, numRetries)
}
delay := policy.Delay("", res, err, numRetries)
if delay != time.Duration(0) {
t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, time.Duration(0), res, err, numRetries)
}
}
}

View file

@ -1,569 +0,0 @@
package aws_test
import (
"fmt"
"github.com/AdRoll/goamz/aws"
"gopkg.in/check.v1"
"net/http"
"strings"
"time"
)
var _ = check.Suite(&V4SignerSuite{})
type V4SignerSuite struct {
auth aws.Auth
region aws.Region
cases []V4SignerSuiteCase
}
type V4SignerSuiteCase struct {
label string
request V4SignerSuiteCaseRequest
canonicalRequest string
stringToSign string
signature string
authorization string
}
type V4SignerSuiteCaseRequest struct {
method string
host string
url string
headers []string
body string
}
func (s *V4SignerSuite) SetUpSuite(c *check.C) {
s.auth = aws.Auth{AccessKey: "AKIDEXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"}
s.region = aws.USEast
// Test cases from the Signature Version 4 Test Suite (http://goo.gl/nguvs0)
s.cases = append(s.cases,
// get-header-key-duplicate
V4SignerSuiteCase{
label: "get-header-key-duplicate",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "ZOO:zoobar", "zoo:foobar", "zoo:zoobar"},
},
canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:foobar,zoobar,zoobar\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n3c52f0eaae2b61329c0a332e3fa15842a37bc5812cf4d80eb64784308850e313",
signature: "54afcaaf45b331f81cd2edb974f7b824ff4dd594cbbaa945ed636b48477368ed",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=54afcaaf45b331f81cd2edb974f7b824ff4dd594cbbaa945ed636b48477368ed",
},
// get-header-value-order
V4SignerSuiteCase{
label: "get-header-value-order",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "p:z", "p:a", "p:p", "p:a"},
},
canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\np:a,a,p,z\n\ndate;host;p\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n94c0389fefe0988cbbedc8606f0ca0b485b48da010d09fc844b45b697c8924fe",
signature: "d2973954263943b11624a11d1c963ca81fb274169c7868b2858c04f083199e3d",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;p, Signature=d2973954263943b11624a11d1c963ca81fb274169c7868b2858c04f083199e3d",
},
// get-header-value-trim
V4SignerSuiteCase{
label: "get-header-value-trim",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "p: phfft "},
},
canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\np:phfft\n\ndate;host;p\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ndddd1902add08da1ac94782b05f9278c08dc7468db178a84f8950d93b30b1f35",
signature: "debf546796015d6f6ded8626f5ce98597c33b47b9164cf6b17b4642036fcb592",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;p, Signature=debf546796015d6f6ded8626f5ce98597c33b47b9164cf6b17b4642036fcb592",
},
// get-empty
V4SignerSuiteCase{
label: "get-relative-relative",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// get-single-relative
V4SignerSuiteCase{
label: "get-relative-relative",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/.",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// get-multiple-relative
V4SignerSuiteCase{
label: "get-relative-relative",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/./././",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// get-relative-relative
V4SignerSuiteCase{
label: "get-relative-relative",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/foo/bar/../..",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// get-relative
V4SignerSuiteCase{
label: "get-relative",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/foo/..",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// get-slash-dot-slash
V4SignerSuiteCase{
label: "get-slash-dot-slash",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/./",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// get-slash-pointless-dot
V4SignerSuiteCase{
label: "get-slash-pointless-dot",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/./foo",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/foo\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n8021a97572ee460f87ca67f4e8c0db763216d84715f5424a843a5312a3321e2d",
signature: "910e4d6c9abafaf87898e1eb4c929135782ea25bb0279703146455745391e63a",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=910e4d6c9abafaf87898e1eb4c929135782ea25bb0279703146455745391e63a",
},
// get-slash
V4SignerSuiteCase{
label: "get-slash",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "//",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// get-slashes
V4SignerSuiteCase{
label: "get-slashes",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "//foo//",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/foo/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n6bb4476ee8745730c9cb79f33a0c70baa6d8af29c0077fa12e4e8f1dd17e7098",
signature: "b00392262853cfe3201e47ccf945601079e9b8a7f51ee4c3d9ee4f187aa9bf19",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b00392262853cfe3201e47ccf945601079e9b8a7f51ee4c3d9ee4f187aa9bf19",
},
// get-space
V4SignerSuiteCase{
label: "get-space",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/%20/foo",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/%20/foo\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n69c45fb9fe3fd76442b5086e50b2e9fec8298358da957b293ef26e506fdfb54b",
signature: "f309cfbd10197a230c42dd17dbf5cca8a0722564cb40a872d25623cfa758e374",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=f309cfbd10197a230c42dd17dbf5cca8a0722564cb40a872d25623cfa758e374",
},
// get-unreserved
V4SignerSuiteCase{
label: "get-unreserved",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ndf63ee3247c0356c696a3b21f8d8490b01fa9cd5bc6550ef5ef5f4636b7b8901",
signature: "830cc36d03f0f84e6ee4953fbe701c1c8b71a0372c63af9255aa364dd183281e",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=830cc36d03f0f84e6ee4953fbe701c1c8b71a0372c63af9255aa364dd183281e",
},
// get-utf8
V4SignerSuiteCase{
label: "get-utf8",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/%E1%88%B4",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/%E1%88%B4\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n27ba31df5dbc6e063d8f87d62eb07143f7f271c5330a917840586ac1c85b6f6b",
signature: "8d6634c189aa8c75c2e51e106b6b5121bed103fdb351f7d7d4381c738823af74",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=8d6634c189aa8c75c2e51e106b6b5121bed103fdb351f7d7d4381c738823af74",
},
// get-vanilla-empty-query-key
V4SignerSuiteCase{
label: "get-vanilla-empty-query-key",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/?foo=bar",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n0846c2945b0832deb7a463c66af5c4f8bd54ec28c438e67a214445b157c9ddf8",
signature: "56c054473fd260c13e4e7393eb203662195f5d4a1fada5314b8b52b23f985e9f",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=56c054473fd260c13e4e7393eb203662195f5d4a1fada5314b8b52b23f985e9f",
},
// get-vanilla-query-order-key-case
V4SignerSuiteCase{
label: "get-vanilla-query-order-key-case",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/?foo=Zoo&foo=aha",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\nfoo=Zoo&foo=aha\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ne25f777ba161a0f1baf778a87faf057187cf5987f17953320e3ca399feb5f00d",
signature: "be7148d34ebccdc6423b19085378aa0bee970bdc61d144bd1a8c48c33079ab09",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=be7148d34ebccdc6423b19085378aa0bee970bdc61d144bd1a8c48c33079ab09",
},
// get-vanilla-query-order-key
V4SignerSuiteCase{
label: "get-vanilla-query-order-key",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/?a=foo&b=foo",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\na=foo&b=foo\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n2f23d14fe13caebf6dfda346285c6d9c14f49eaca8f5ec55c627dd7404f7a727",
signature: "0dc122f3b28b831ab48ba65cb47300de53fbe91b577fe113edac383730254a3b",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=0dc122f3b28b831ab48ba65cb47300de53fbe91b577fe113edac383730254a3b",
},
// get-vanilla-query-order-value
V4SignerSuiteCase{
label: "get-vanilla-query-order-value",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/?foo=b&foo=a",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\nfoo=a&foo=b\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n33dffc220e89131f8f6157a35c40903daa658608d9129ff9489e5cf5bbd9b11b",
signature: "feb926e49e382bec75c9d7dcb2a1b6dc8aa50ca43c25d2bc51143768c0875acc",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=feb926e49e382bec75c9d7dcb2a1b6dc8aa50ca43c25d2bc51143768c0875acc",
},
// get-vanilla-query-unreserved
V4SignerSuiteCase{
label: "get-vanilla-query-unreserved",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/?-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nd2578f3156d4c9d180713d1ff20601d8a3eed0dd35447d24603d7d67414bd6b5",
signature: "f1498ddb4d6dae767d97c466fb92f1b59a2c71ca29ac954692663f9db03426fb",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=f1498ddb4d6dae767d97c466fb92f1b59a2c71ca29ac954692663f9db03426fb",
},
// get-vanilla-query
V4SignerSuiteCase{
label: "get-vanilla-query",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// get-vanilla-ut8-query
V4SignerSuiteCase{
label: "get-vanilla-ut8-query",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/?ሴ=bar",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n%E1%88%B4=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nde5065ff39c131e6c2e2bd19cd9345a794bf3b561eab20b8d97b2093fc2a979e",
signature: "6fb359e9a05394cc7074e0feb42573a2601abc0c869a953e8c5c12e4e01f1a8c",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=6fb359e9a05394cc7074e0feb42573a2601abc0c869a953e8c5c12e4e01f1a8c",
},
// get-vanilla
V4SignerSuiteCase{
label: "get-vanilla",
request: V4SignerSuiteCaseRequest{
method: "GET",
host: "host.foo.com",
url: "/",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
},
// post-header-key-case
V4SignerSuiteCase{
label: "post-header-key-case",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n05da62cee468d24ae84faff3c39f1b85540de60243c1bcaace39c0a2acc7b2c4",
signature: "22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726",
},
// post-header-key-sort
V4SignerSuiteCase{
label: "post-header-key-sort",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "ZOO:zoobar"},
},
canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:zoobar\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n34e1bddeb99e76ee01d63b5e28656111e210529efeec6cdfd46a48e4c734545d",
signature: "b7a95a52518abbca0964a999a880429ab734f35ebbf1235bd79a5de87756dc4a",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=b7a95a52518abbca0964a999a880429ab734f35ebbf1235bd79a5de87756dc4a",
},
// post-header-value-case
V4SignerSuiteCase{
label: "post-header-value-case",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "zoo:ZOOBAR"},
},
canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:ZOOBAR\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n3aae6d8274b8c03e2cc96fc7d6bda4b9bd7a0a184309344470b2c96953e124aa",
signature: "273313af9d0c265c531e11db70bbd653f3ba074c1009239e8559d3987039cad7",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=273313af9d0c265c531e11db70bbd653f3ba074c1009239e8559d3987039cad7",
},
// post-vanilla-empty-query-value
V4SignerSuiteCase{
label: "post-vanilla-empty-query-value",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/?foo=bar",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "POST\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ncd4f39132d8e60bb388831d734230460872b564871c47f5de62e62d1a68dbe1e",
signature: "b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92",
},
// post-vanilla-query
V4SignerSuiteCase{
label: "post-vanilla-query",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/?foo=bar",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "POST\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ncd4f39132d8e60bb388831d734230460872b564871c47f5de62e62d1a68dbe1e",
signature: "b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92",
},
// post-vanilla
V4SignerSuiteCase{
label: "post-vanilla",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
},
canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n05da62cee468d24ae84faff3c39f1b85540de60243c1bcaace39c0a2acc7b2c4",
signature: "22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726",
},
// post-x-www-form-urlencoded-parameters
V4SignerSuiteCase{
label: "post-x-www-form-urlencoded-parameters",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"Content-Type:application/x-www-form-urlencoded; charset=utf8", "Date:Mon, 09 Sep 2011 23:36:00 GMT"},
body: "foo=bar",
},
canonicalRequest: "POST\n/\n\ncontent-type:application/x-www-form-urlencoded; charset=utf8\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ncontent-type;date;host\n3ba8907e7a252327488df390ed517c45b96dead033600219bdca7107d1d3f88a",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nc4115f9e54b5cecf192b1eaa23b8e88ed8dc5391bd4fde7b3fff3d9c9fe0af1f",
signature: "b105eb10c6d318d2294de9d49dd8b031b55e3c3fe139f2e637da70511e9e7b71",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=content-type;date;host, Signature=b105eb10c6d318d2294de9d49dd8b031b55e3c3fe139f2e637da70511e9e7b71",
},
// post-x-www-form-urlencoded
V4SignerSuiteCase{
label: "post-x-www-form-urlencoded",
request: V4SignerSuiteCaseRequest{
method: "POST",
host: "host.foo.com",
url: "/",
headers: []string{"Content-Type:application/x-www-form-urlencoded", "Date:Mon, 09 Sep 2011 23:36:00 GMT"},
body: "foo=bar",
},
canonicalRequest: "POST\n/\n\ncontent-type:application/x-www-form-urlencoded\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ncontent-type;date;host\n3ba8907e7a252327488df390ed517c45b96dead033600219bdca7107d1d3f88a",
stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n4c5c6e4b52fb5fb947a8733982a8a5a61b14f04345cbfe6e739236c76dd48f74",
signature: "5a15b22cf462f047318703b92e6f4f38884e4a7ab7b1d6426ca46a8bd1c26cbc",
authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=content-type;date;host, Signature=5a15b22cf462f047318703b92e6f4f38884e4a7ab7b1d6426ca46a8bd1c26cbc",
},
)
}
func (s *V4SignerSuite) TestCases(c *check.C) {
signer := aws.NewV4Signer(s.auth, "host", s.region)
for _, testCase := range s.cases {
req, err := http.NewRequest(testCase.request.method, "http://"+testCase.request.host+testCase.request.url, strings.NewReader(testCase.request.body))
c.Assert(err, check.IsNil, check.Commentf("Testcase: %s", testCase.label))
for _, v := range testCase.request.headers {
h := strings.SplitN(v, ":", 2)
req.Header.Add(h[0], h[1])
}
req.Header.Set("host", req.Host)
t := signer.RequestTime(req)
canonicalRequest := signer.CanonicalRequest(req)
c.Check(canonicalRequest, check.Equals, testCase.canonicalRequest, check.Commentf("Testcase: %s", testCase.label))
stringToSign := signer.StringToSign(t, canonicalRequest)
c.Check(stringToSign, check.Equals, testCase.stringToSign, check.Commentf("Testcase: %s", testCase.label))
signature := signer.Signature(t, stringToSign)
c.Check(signature, check.Equals, testCase.signature, check.Commentf("Testcase: %s", testCase.label))
authorization := signer.Authorization(req.Header, t, signature)
c.Check(authorization, check.Equals, testCase.authorization, check.Commentf("Testcase: %s", testCase.label))
signer.Sign(req)
c.Check(req.Header.Get("Authorization"), check.Equals, testCase.authorization, check.Commentf("Testcase: %s", testCase.label))
}
}
func ExampleV4Signer() {
// Get auth from env vars
auth, err := aws.EnvAuth()
if err != nil {
fmt.Println(err)
}
// Create a signer with the auth, name of the service, and aws region
signer := aws.NewV4Signer(auth, "dynamodb", aws.USEast)
// Create a request
req, err := http.NewRequest("POST", aws.USEast.DynamoDBEndpoint, strings.NewReader("sample_request"))
if err != nil {
fmt.Println(err)
}
// Date or x-amz-date header is required to sign a request
req.Header.Add("Date", time.Now().UTC().Format(http.TimeFormat))
// Sign the request
signer.Sign(req)
// Issue signed request
http.DefaultClient.Do(req)
}

View file

@ -1,52 +0,0 @@
package cloudfront
import (
"crypto/x509"
"encoding/pem"
"io/ioutil"
"net/url"
"testing"
"time"
)
func TestSignedCannedURL(t *testing.T) {
rawKey, err := ioutil.ReadFile("testdata/key.pem")
if err != nil {
t.Fatal(err)
}
pemKey, _ := pem.Decode(rawKey)
privateKey, err := x509.ParsePKCS1PrivateKey(pemKey.Bytes)
if err != nil {
t.Fatal(err)
}
cf := &CloudFront{
key: privateKey,
keyPairId: "test-key-pair-1231245",
BaseURL: "https://cloudfront.com",
}
expireTime, err := time.Parse(time.RFC3339, "2014-03-28T14:00:21Z")
if err != nil {
t.Fatal(err)
}
query := make(url.Values)
query.Add("test", "value")
uri, err := cf.CannedSignedURL("test", "test=value", expireTime)
if err != nil {
t.Fatal(err)
}
parsed, err := url.Parse(uri)
if err != nil {
t.Fatal(err)
}
signature := parsed.Query().Get("Signature")
if signature == "" {
t.Fatal("Encoded signature is empty")
}
}

View file

@ -1,6 +0,0 @@
-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC0yMzp9DkPAE99DhsEaGkqougL
vtmDKri4bZj0fFjmGmjyyjz9hlrsr87LHVWzH/7igK7040HG1UqypX3ijtJa9+6B
KHwBBctboU3y4GfwFwVAOumY9UytFpyPlgUFrffZLQAywKkT24OgcfEj0G5kiQn7
60wFnmSUtOuITo708QIDAQAB
-----END PUBLIC KEY-----

View file

@ -1,27 +0,0 @@
package s3
import (
"github.com/AdRoll/goamz/aws"
)
var originalStrategy = attempts
func SetAttemptStrategy(s *aws.AttemptStrategy) {
if s == nil {
attempts = originalStrategy
} else {
attempts = *s
}
}
func Sign(auth aws.Auth, method, path string, params, headers map[string][]string) {
sign(auth, method, path, params, headers)
}
func SetListPartsMax(n int) {
listPartsMax = n
}
func SetListMultiMax(n int) {
listMultiMax = n
}

View file

@ -1,205 +0,0 @@
package s3_test
import (
"encoding/xml"
"github.com/AdRoll/goamz/s3"
"gopkg.in/check.v1"
"io/ioutil"
"net/http"
"strings"
"time"
)
func (s *S) TestLifecycleConfiguration(c *check.C) {
date, err := time.Parse(s3.LifecycleRuleDateFormat, "2014-09-10")
c.Check(err, check.IsNil)
conf := &s3.LifecycleConfiguration{}
rule := s3.NewLifecycleRule("transition-days", "/")
rule.SetTransitionDays(7)
conf.AddRule(rule)
rule = s3.NewLifecycleRule("transition-date", "/")
rule.SetTransitionDate(date)
conf.AddRule(rule)
rule = s3.NewLifecycleRule("expiration-days", "")
rule.SetExpirationDays(1)
conf.AddRule(rule)
rule = s3.NewLifecycleRule("expiration-date", "")
rule.SetExpirationDate(date)
conf.AddRule(rule)
rule = s3.NewLifecycleRule("noncurrent-transition", "")
rule.SetNoncurrentVersionTransitionDays(11)
conf.AddRule(rule)
rule = s3.NewLifecycleRule("noncurrent-expiration", "")
rule.SetNoncurrentVersionExpirationDays(1011)
// Test Disable() and Enable() toggling
c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusEnabled)
rule.Disable()
c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusDisabled)
rule.Enable()
c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusEnabled)
rule.Disable()
c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusDisabled)
conf.AddRule(rule)
doc, err := xml.MarshalIndent(conf, "", " ")
c.Check(err, check.IsNil)
expectedDoc := `<LifecycleConfiguration>
<Rule>
<ID>transition-days</ID>
<Prefix>/</Prefix>
<Status>Enabled</Status>
<Transition>
<Days>7</Days>
<StorageClass>GLACIER</StorageClass>
</Transition>
</Rule>
<Rule>
<ID>transition-date</ID>
<Prefix>/</Prefix>
<Status>Enabled</Status>
<Transition>
<Date>2014-09-10</Date>
<StorageClass>GLACIER</StorageClass>
</Transition>
</Rule>
<Rule>
<ID>expiration-days</ID>
<Prefix></Prefix>
<Status>Enabled</Status>
<Expiration>
<Days>1</Days>
</Expiration>
</Rule>
<Rule>
<ID>expiration-date</ID>
<Prefix></Prefix>
<Status>Enabled</Status>
<Expiration>
<Date>2014-09-10</Date>
</Expiration>
</Rule>
<Rule>
<ID>noncurrent-transition</ID>
<Prefix></Prefix>
<Status>Enabled</Status>
<NoncurrentVersionTransition>
<NoncurrentDays>11</NoncurrentDays>
<StorageClass>GLACIER</StorageClass>
</NoncurrentVersionTransition>
</Rule>
<Rule>
<ID>noncurrent-expiration</ID>
<Prefix></Prefix>
<Status>Disabled</Status>
<NoncurrentVersionExpiration>
<NoncurrentDays>1011</NoncurrentDays>
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>`
c.Check(string(doc), check.Equals, expectedDoc)
// Unmarshalling test
conf2 := &s3.LifecycleConfiguration{}
err = xml.Unmarshal(doc, conf2)
c.Check(err, check.IsNil)
s.checkLifecycleConfigurationEqual(c, conf, conf2)
}
func (s *S) checkLifecycleConfigurationEqual(c *check.C, conf, conf2 *s3.LifecycleConfiguration) {
c.Check(len(*conf2.Rules), check.Equals, len(*conf.Rules))
for i, rule := range *conf2.Rules {
confRules := *conf.Rules
c.Check(rule, check.DeepEquals, confRules[i])
}
}
func (s *S) checkLifecycleRequest(c *check.C, req *http.Request) {
// ?lifecycle= is the only query param
v, ok := req.Form["lifecycle"]
c.Assert(ok, check.Equals, true)
c.Assert(v, check.HasLen, 1)
c.Assert(v[0], check.Equals, "")
c.Assert(req.Header["X-Amz-Date"], check.HasLen, 1)
c.Assert(req.Header["X-Amz-Date"][0], check.Not(check.Equals), "")
// Lifecycle methods require V4 auth
usesV4 := strings.HasPrefix(req.Header["Authorization"][0], "AWS4-HMAC-SHA256")
c.Assert(usesV4, check.Equals, true)
}
func (s *S) TestPutLifecycleConfiguration(c *check.C) {
testServer.Response(200, nil, "")
conf := &s3.LifecycleConfiguration{}
rule := s3.NewLifecycleRule("id", "")
rule.SetTransitionDays(7)
conf.AddRule(rule)
doc, err := xml.Marshal(conf)
c.Check(err, check.IsNil)
b := s.s3.Bucket("bucket")
err = b.PutLifecycleConfiguration(conf)
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/bucket/")
c.Assert(req.Header["Content-Md5"], check.HasLen, 1)
c.Assert(req.Header["Content-Md5"][0], check.Not(check.Equals), "")
s.checkLifecycleRequest(c, req)
// Check we sent the correct xml serialization
data, err := ioutil.ReadAll(req.Body)
req.Body.Close()
c.Assert(err, check.IsNil)
header := "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
c.Assert(string(data), check.Equals, header+string(doc))
}
func (s *S) TestGetLifecycleConfiguration(c *check.C) {
conf := &s3.LifecycleConfiguration{}
rule := s3.NewLifecycleRule("id", "")
rule.SetTransitionDays(7)
conf.AddRule(rule)
doc, err := xml.Marshal(conf)
c.Check(err, check.IsNil)
testServer.Response(200, nil, string(doc))
b := s.s3.Bucket("bucket")
conf2, err := b.GetLifecycleConfiguration()
c.Check(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/bucket/")
s.checkLifecycleRequest(c, req)
s.checkLifecycleConfigurationEqual(c, conf, conf2)
}
func (s *S) TestDeleteLifecycleConfiguration(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
err := b.DeleteLifecycleConfiguration()
c.Check(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "DELETE")
c.Assert(req.URL.Path, check.Equals, "/bucket/")
s.checkLifecycleRequest(c, req)
}

View file

@ -1,480 +0,0 @@
package s3_test
import (
"encoding/xml"
"io"
"io/ioutil"
"strings"
"github.com/AdRoll/goamz/s3"
"gopkg.in/check.v1"
)
func (s *S) TestInitMulti(c *check.C) {
testServer.Response(200, nil, InitMultiResultDump)
b := s.s3.Bucket("sample")
metadata := make(map[string][]string)
metadata["key1"] = []string{"value1"}
metadata["key2"] = []string{"value2"}
options := s3.Options{
SSE: true,
Meta: metadata,
ContentEncoding: "text/utf8",
CacheControl: "no-cache",
RedirectLocation: "http://github.com/AdRoll/goamz",
ContentMD5: "0000000000000000",
}
multi, err := b.InitMulti("multi", "text/plain", s3.Private, options)
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "POST")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"text/plain"})
c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
c.Assert(req.Header["X-Amz-Server-Side-Encryption"], check.DeepEquals, []string{"AES256"})
c.Assert(req.Header["Content-Encoding"], check.DeepEquals, []string{"text/utf8"})
c.Assert(req.Header["Cache-Control"], check.DeepEquals, []string{"no-cache"})
c.Assert(req.Header["Content-Md5"], check.DeepEquals, []string{"0000000000000000"})
c.Assert(req.Header["X-Amz-Website-Redirect-Location"], check.DeepEquals, []string{"http://github.com/AdRoll/goamz"})
c.Assert(req.Header["X-Amz-Meta-Key1"], check.DeepEquals, []string{"value1"})
c.Assert(req.Header["X-Amz-Meta-Key2"], check.DeepEquals, []string{"value2"})
c.Assert(multi.UploadId, check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
}
func (s *S) TestMultiNoPreviousUpload(c *check.C) {
// Don't retry the NoSuchUpload error.
s.DisableRetries()
testServer.Response(404, nil, NoSuchUploadErrorDump)
testServer.Response(200, nil, InitMultiResultDump)
b := s.s3.Bucket("sample")
multi, err := b.Multi("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/sample/")
c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
c.Assert(req.Form["prefix"], check.DeepEquals, []string{"multi"})
req = testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "POST")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
c.Assert(multi.UploadId, check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
}
func (s *S) TestMultiReturnOld(c *check.C) {
testServer.Response(200, nil, ListMultiResultDump)
b := s.s3.Bucket("sample")
multi, err := b.Multi("multi1", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
c.Assert(multi.Key, check.Equals, "multi1")
c.Assert(multi.UploadId, check.Equals, "iUVug89pPvSswrikD")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/sample/")
c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
c.Assert(req.Form["prefix"], check.DeepEquals, []string{"multi1"})
}
func (s *S) TestListParts(c *check.C) {
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(200, nil, ListPartsResultDump1)
testServer.Response(404, nil, NoSuchUploadErrorDump) // :-(
testServer.Response(200, nil, ListPartsResultDump2)
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
parts, err := multi.ListParts()
c.Assert(err, check.IsNil)
c.Assert(parts, check.HasLen, 3)
c.Assert(parts[0].N, check.Equals, 1)
c.Assert(parts[0].Size, check.Equals, int64(5))
c.Assert(parts[0].ETag, check.Equals, `"ffc88b4ca90a355f8ddba6b2c3b2af5c"`)
c.Assert(parts[1].N, check.Equals, 2)
c.Assert(parts[1].Size, check.Equals, int64(5))
c.Assert(parts[1].ETag, check.Equals, `"d067a0fa9dc61a6e7195ca99696b5a89"`)
c.Assert(parts[2].N, check.Equals, 3)
c.Assert(parts[2].Size, check.Equals, int64(5))
c.Assert(parts[2].ETag, check.Equals, `"49dcd91231f801159e893fb5c6674985"`)
testServer.WaitRequest()
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
c.Assert(req.Form["max-parts"], check.DeepEquals, []string{"1000"})
testServer.WaitRequest() // The internal error.
req = testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
c.Assert(req.Form["max-parts"], check.DeepEquals, []string{"1000"})
c.Assert(req.Form["part-number-marker"], check.DeepEquals, []string{"2"})
}
func (s *S) TestPutPart(c *check.C) {
headers := map[string]string{
"ETag": `"26f90efd10d614f100252ff56d88dad8"`,
}
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(200, headers, "")
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
part, err := multi.PutPart(1, strings.NewReader("<part 1>"))
c.Assert(err, check.IsNil)
c.Assert(part.N, check.Equals, 1)
c.Assert(part.Size, check.Equals, int64(8))
c.Assert(part.ETag, check.Equals, headers["ETag"])
testServer.WaitRequest()
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"8"})
c.Assert(req.Header["Content-Md5"], check.DeepEquals, []string{"JvkO/RDWFPEAJS/1bYja2A=="})
}
func (s *S) TestPutPartCopy(c *check.C) {
testServer.Response(200, nil, InitMultiResultDump)
// PutPartCopy makes a Head request internally to verify access to the source object
// and obtain its size
testServer.Response(200, nil, "content")
testServer.Response(200, nil, PutCopyResultDump)
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
res, part, err := multi.PutPartCopy(1, s3.CopyOptions{}, "source-bucket/\u00FCber-fil\u00E9.jpg")
c.Assert(err, check.IsNil)
c.Assert(part.N, check.Equals, 1)
c.Assert(part.Size, check.Equals, int64(7))
c.Assert(res, check.DeepEquals, &s3.CopyObjectResult{
ETag: `"9b2cf535f27731c974343645a3985328"`,
LastModified: `2009-10-28T22:32:00`})
// Verify the Head request
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "POST")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
c.Assert(err, check.IsNil)
testServer.WaitRequest()
req = testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"})
c.Assert(req.Header["X-Amz-Copy-Source"], check.DeepEquals, []string{`source-bucket%2F%C3%BCber-fil%C3%A9.jpg`})
}
func readAll(r io.Reader) string {
data, err := ioutil.ReadAll(r)
if err != nil {
panic(err)
}
return string(data)
}
func (s *S) TestPutAllNoPreviousUpload(c *check.C) {
// Don't retry the NoSuchUpload error.
s.DisableRetries()
etag1 := map[string]string{"ETag": `"etag1"`}
etag2 := map[string]string{"ETag": `"etag2"`}
etag3 := map[string]string{"ETag": `"etag3"`}
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(404, nil, NoSuchUploadErrorDump)
testServer.Response(200, etag1, "")
testServer.Response(200, etag2, "")
testServer.Response(200, etag3, "")
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
parts, err := multi.PutAll(strings.NewReader("part1part2last"), 5)
c.Assert(parts, check.HasLen, 3)
c.Assert(parts[0].ETag, check.Equals, `"etag1"`)
c.Assert(parts[1].ETag, check.Equals, `"etag2"`)
c.Assert(parts[2].ETag, check.Equals, `"etag3"`)
c.Assert(err, check.IsNil)
// Init
testServer.WaitRequest()
// List old parts. Won't find anything.
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
// Send part 1.
req = testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"})
c.Assert(readAll(req.Body), check.Equals, "part1")
// Send part 2.
req = testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"2"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"})
c.Assert(readAll(req.Body), check.Equals, "part2")
// Send part 3 with shorter body.
req = testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"3"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"4"})
c.Assert(readAll(req.Body), check.Equals, "last")
}
func (s *S) TestPutAllZeroSizeFile(c *check.C) {
// Don't retry the NoSuchUpload error.
s.DisableRetries()
etag1 := map[string]string{"ETag": `"etag1"`}
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(404, nil, NoSuchUploadErrorDump)
testServer.Response(200, etag1, "")
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
// Must send at least one part, so that completing it will work.
parts, err := multi.PutAll(strings.NewReader(""), 5)
c.Assert(parts, check.HasLen, 1)
c.Assert(parts[0].ETag, check.Equals, `"etag1"`)
c.Assert(err, check.IsNil)
// Init
testServer.WaitRequest()
// List old parts. Won't find anything.
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
// Send empty part.
req = testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"0"})
c.Assert(readAll(req.Body), check.Equals, "")
}
func (s *S) TestPutAllResume(c *check.C) {
etag2 := map[string]string{"ETag": `"etag2"`}
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(200, nil, ListPartsResultDump1)
testServer.Response(200, nil, ListPartsResultDump2)
testServer.Response(200, etag2, "")
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
// "part1" and "part3" match the checksums in ResultDump1.
// The middle one is a mismatch (it refers to "part2").
parts, err := multi.PutAll(strings.NewReader("part1partXpart3"), 5)
c.Assert(parts, check.HasLen, 3)
c.Assert(parts[0].N, check.Equals, 1)
c.Assert(parts[0].Size, check.Equals, int64(5))
c.Assert(parts[0].ETag, check.Equals, `"ffc88b4ca90a355f8ddba6b2c3b2af5c"`)
c.Assert(parts[1].N, check.Equals, 2)
c.Assert(parts[1].Size, check.Equals, int64(5))
c.Assert(parts[1].ETag, check.Equals, `"etag2"`)
c.Assert(parts[2].N, check.Equals, 3)
c.Assert(parts[2].Size, check.Equals, int64(5))
c.Assert(parts[2].ETag, check.Equals, `"49dcd91231f801159e893fb5c6674985"`)
c.Assert(err, check.IsNil)
// Init
testServer.WaitRequest()
// List old parts, broken in two requests.
for i := 0; i < 2; i++ {
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
}
// Send part 2, as it didn't match the checksum.
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"2"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"})
c.Assert(readAll(req.Body), check.Equals, "partX")
}
func (s *S) TestMultiComplete(c *check.C) {
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(200, nil, MultiCompleteDump)
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}})
c.Assert(err, check.IsNil)
testServer.WaitRequest()
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "POST")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
var payload struct {
XMLName xml.Name
Part []struct {
PartNumber int
ETag string
}
}
dec := xml.NewDecoder(req.Body)
err = dec.Decode(&payload)
c.Assert(err, check.IsNil)
c.Assert(payload.XMLName.Local, check.Equals, "CompleteMultipartUpload")
c.Assert(len(payload.Part), check.Equals, 2)
c.Assert(payload.Part[0].PartNumber, check.Equals, 1)
c.Assert(payload.Part[0].ETag, check.Equals, `"ETag1"`)
c.Assert(payload.Part[1].PartNumber, check.Equals, 2)
c.Assert(payload.Part[1].ETag, check.Equals, `"ETag2"`)
}
func (s *S) TestMultiCompleteError(c *check.C) {
testServer.Response(200, nil, InitMultiResultDump)
// Note the 200 response. Completing will hold the connection on some
// kind of long poll, and may return a late error even after a 200.
testServer.Response(200, nil, InternalErrorDump)
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}})
c.Assert(err, check.NotNil)
testServer.WaitRequest()
testServer.WaitRequest()
}
func (s *S) TestMultiAbort(c *check.C) {
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(200, nil, "")
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
err = multi.Abort()
c.Assert(err, check.IsNil)
testServer.WaitRequest()
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "DELETE")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
}
func (s *S) TestListMulti(c *check.C) {
testServer.Response(200, nil, ListMultiResultDump)
b := s.s3.Bucket("sample")
multis, prefixes, err := b.ListMulti("", "/")
c.Assert(err, check.IsNil)
c.Assert(prefixes, check.DeepEquals, []string{"a/", "b/"})
c.Assert(multis, check.HasLen, 2)
c.Assert(multis[0].Key, check.Equals, "multi1")
c.Assert(multis[0].UploadId, check.Equals, "iUVug89pPvSswrikD")
c.Assert(multis[1].Key, check.Equals, "multi2")
c.Assert(multis[1].UploadId, check.Equals, "DkirwsSvPp98guVUi")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/sample/")
c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
c.Assert(req.Form["prefix"], check.DeepEquals, []string{""})
c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"})
c.Assert(req.Form["max-uploads"], check.DeepEquals, []string{"1000"})
}
func (s *S) TestMultiCompleteSupportRadosGW(c *check.C) {
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(200, nil, MultiCompleteDump)
s.s3.Region.Name = "generic"
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}})
c.Assert(err, check.IsNil)
testServer.WaitRequest()
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "POST")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
c.Assert(req.Header["Content-Length"], check.NotNil)
var payload struct {
XMLName xml.Name
Part []struct {
PartNumber int
ETag string
}
}
dec := xml.NewDecoder(req.Body)
err = dec.Decode(&payload)
c.Assert(err, check.IsNil)
c.Assert(payload.XMLName.Local, check.Equals, "CompleteMultipartUpload")
c.Assert(len(payload.Part), check.Equals, 2)
c.Assert(payload.Part[0].PartNumber, check.Equals, 1)
c.Assert(payload.Part[0].ETag, check.Equals, `"ETag1"`)
c.Assert(payload.Part[1].PartNumber, check.Equals, 2)
c.Assert(payload.Part[1].ETag, check.Equals, `"ETag2"`)
}

View file

@ -1,248 +0,0 @@
package s3_test
var PutCopyResultDump = `
<?xml version="1.0" encoding="UTF-8"?>
<CopyObjectResult>
<LastModified>2009-10-28T22:32:00</LastModified>
<ETag>&quot;9b2cf535f27731c974343645a3985328&quot;</ETag>
</CopyObjectResult>
`
var GetObjectErrorDump = `
<?xml version="1.0" encoding="UTF-8"?>
<Error><Code>NoSuchBucket</Code><Message>The specified bucket does not exist</Message>
<BucketName>non-existent-bucket</BucketName><RequestId>3F1B667FAD71C3D8</RequestId>
<HostId>L4ee/zrm1irFXY5F45fKXIRdOf9ktsKY/8TDVawuMK2jWRb1RF84i1uBzkdNqS5D</HostId></Error>
`
var GetListResultDump1 = `
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
<Name>quotes</Name>
<Prefix>N</Prefix>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>Nelson</Key>
<LastModified>2006-01-01T12:00:00.000Z</LastModified>
<ETag>&quot;828ef3fdfa96f00ad9f27c383fc9ac7f&quot;</ETag>
<Size>5</Size>
<StorageClass>STANDARD</StorageClass>
<Owner>
<ID>bcaf161ca5fb16fd081034f</ID>
<DisplayName>webfile</DisplayName>
</Owner>
</Contents>
<Contents>
<Key>Neo</Key>
<LastModified>2006-01-01T12:00:00.000Z</LastModified>
<ETag>&quot;828ef3fdfa96f00ad9f27c383fc9ac7f&quot;</ETag>
<Size>4</Size>
<StorageClass>STANDARD</StorageClass>
<Owner>
<ID>bcaf1ffd86a5fb16fd081034f</ID>
<DisplayName>webfile</DisplayName>
</Owner>
</Contents>
</ListBucketResult>
`
var GetListResultDump2 = `
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>example-bucket</Name>
<Prefix>photos/2006/</Prefix>
<Marker>some-marker</Marker>
<MaxKeys>1000</MaxKeys>
<Delimiter>/</Delimiter>
<IsTruncated>false</IsTruncated>
<CommonPrefixes>
<Prefix>photos/2006/feb/</Prefix>
</CommonPrefixes>
<CommonPrefixes>
<Prefix>photos/2006/jan/</Prefix>
</CommonPrefixes>
</ListBucketResult>
`
var InitMultiResultDump = `
<?xml version="1.0" encoding="UTF-8"?>
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>sample</Bucket>
<Key>multi</Key>
<UploadId>JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ--</UploadId>
</InitiateMultipartUploadResult>
`
var ListPartsResultDump1 = `
<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>sample</Bucket>
<Key>multi</Key>
<UploadId>JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ--</UploadId>
<Initiator>
<ID>bb5c0f63b0b25f2d099c</ID>
<DisplayName>joe</DisplayName>
</Initiator>
<Owner>
<ID>bb5c0f63b0b25f2d099c</ID>
<DisplayName>joe</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
<PartNumberMarker>0</PartNumberMarker>
<NextPartNumberMarker>2</NextPartNumberMarker>
<MaxParts>2</MaxParts>
<IsTruncated>true</IsTruncated>
<Part>
<PartNumber>1</PartNumber>
<LastModified>2013-01-30T13:45:51.000Z</LastModified>
<ETag>&quot;ffc88b4ca90a355f8ddba6b2c3b2af5c&quot;</ETag>
<Size>5</Size>
</Part>
<Part>
<PartNumber>2</PartNumber>
<LastModified>2013-01-30T13:45:52.000Z</LastModified>
<ETag>&quot;d067a0fa9dc61a6e7195ca99696b5a89&quot;</ETag>
<Size>5</Size>
</Part>
</ListPartsResult>
`
var ListPartsResultDump2 = `
<?xml version="1.0" encoding="UTF-8"?>
<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>sample</Bucket>
<Key>multi</Key>
<UploadId>JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ--</UploadId>
<Initiator>
<ID>bb5c0f63b0b25f2d099c</ID>
<DisplayName>joe</DisplayName>
</Initiator>
<Owner>
<ID>bb5c0f63b0b25f2d099c</ID>
<DisplayName>joe</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
<PartNumberMarker>2</PartNumberMarker>
<NextPartNumberMarker>3</NextPartNumberMarker>
<MaxParts>2</MaxParts>
<IsTruncated>false</IsTruncated>
<Part>
<PartNumber>3</PartNumber>
<LastModified>2013-01-30T13:46:50.000Z</LastModified>
<ETag>&quot;49dcd91231f801159e893fb5c6674985&quot;</ETag>
<Size>5</Size>
</Part>
</ListPartsResult>
`
var ListMultiResultDump = `
<?xml version="1.0"?>
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>goamz-test-bucket-us-east-1-akiajk3wyewhctyqbf7a</Bucket>
<KeyMarker/>
<UploadIdMarker/>
<NextKeyMarker>multi1</NextKeyMarker>
<NextUploadIdMarker>iUVug89pPvSswrikD72p8uO62EzhNtpDxRmwC5WSiWDdK9SfzmDqe3xpP1kMWimyimSnz4uzFc3waVM5ufrKYQ--</NextUploadIdMarker>
<Delimiter>/</Delimiter>
<MaxUploads>1000</MaxUploads>
<IsTruncated>false</IsTruncated>
<Upload>
<Key>multi1</Key>
<UploadId>iUVug89pPvSswrikD</UploadId>
<Initiator>
<ID>bb5c0f63b0b25f2d0</ID>
<DisplayName>gustavoniemeyer</DisplayName>
</Initiator>
<Owner>
<ID>bb5c0f63b0b25f2d0</ID>
<DisplayName>gustavoniemeyer</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
<Initiated>2013-01-30T18:15:47.000Z</Initiated>
</Upload>
<Upload>
<Key>multi2</Key>
<UploadId>DkirwsSvPp98guVUi</UploadId>
<Initiator>
<ID>bb5c0f63b0b25f2d0</ID>
<DisplayName>joe</DisplayName>
</Initiator>
<Owner>
<ID>bb5c0f63b0b25f2d0</ID>
<DisplayName>joe</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
<Initiated>2013-01-30T18:15:47.000Z</Initiated>
</Upload>
<CommonPrefixes>
<Prefix>a/</Prefix>
</CommonPrefixes>
<CommonPrefixes>
<Prefix>b/</Prefix>
</CommonPrefixes>
</ListMultipartUploadsResult>
`
var NoSuchUploadErrorDump = `
<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>NoSuchUpload</Code>
<Message>Not relevant</Message>
<BucketName>sample</BucketName>
<RequestId>3F1B667FAD71C3D8</RequestId>
<HostId>kjhwqk</HostId>
</Error>
`
var MultiCompleteDump = `
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Location>http://Example-Bucket.s3.amazonaws.com/Example-Object</Location>
<Bucket>Example-Bucket</Bucket>
<Key>Example-Object</Key>
<ETag>"3858f62230ac3c915f300c664312c11f-9"</ETag>
</CompleteMultipartUploadResult>
`
var InternalErrorDump = `
<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>InternalError</Code>
<Message>Not relevant</Message>
<BucketName>sample</BucketName>
<RequestId>3F1B667FAD71C3D8</RequestId>
<HostId>kjhwqk</HostId>
</Error>
`
var GetServiceDump = `
<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
<Owner>
<ID>bcaf1ffd86f461ca5fb16fd081034f</ID>
<DisplayName>webfile</DisplayName>
</Owner>
<Buckets>
<Bucket>
<Name>quotes</Name>
<CreationDate>2006-02-03T16:45:09.000Z</CreationDate>
</Bucket>
<Bucket>
<Name>samples</Name>
<CreationDate>2006-02-03T16:41:58.000Z</CreationDate>
</Bucket>
</Buckets>
</ListAllMyBucketsResult>
`
var GetLocationUsStandard = `
<?xml version="1.0" encoding="UTF-8"?>
<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>
`
var GetLocationUsWest1 = `
<?xml version="1.0" encoding="UTF-8"?>
<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-west-1</LocationConstraint>
`
var BucketWebsiteConfigurationDump = `<?xml version="1.0" encoding="UTF-8"?>
<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><RedirectAllRequestsTo><HostName>example.com</HostName></RedirectAllRequestsTo></WebsiteConfiguration>`

View file

@ -1,513 +0,0 @@
package s3_test
import (
"bytes"
"io/ioutil"
"net/http"
"testing"
"time"
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"github.com/AdRoll/goamz/testutil"
"gopkg.in/check.v1"
)
func Test(t *testing.T) {
check.TestingT(t)
}
type S struct {
s3 *s3.S3
}
var _ = check.Suite(&S{})
var testServer = testutil.NewHTTPServer()
func (s *S) SetUpSuite(c *check.C) {
testServer.Start()
auth := aws.Auth{AccessKey: "abc", SecretKey: "123"}
s.s3 = s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL})
}
func (s *S) TearDownSuite(c *check.C) {
s3.SetAttemptStrategy(nil)
}
func (s *S) SetUpTest(c *check.C) {
attempts := aws.AttemptStrategy{
Total: 300 * time.Millisecond,
Delay: 100 * time.Millisecond,
}
s3.SetAttemptStrategy(&attempts)
}
func (s *S) TearDownTest(c *check.C) {
testServer.Flush()
}
func (s *S) DisableRetries() {
s3.SetAttemptStrategy(&aws.AttemptStrategy{})
}
// PutBucket docs: http://goo.gl/kBTCu
func (s *S) TestPutBucket(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
err := b.PutBucket(s3.Private)
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/bucket/")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
}
// PutBucketWebsite docs: http://goo.gl/TpRlUy
func (s *S) TestPutBucketWebsite(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
config := s3.WebsiteConfiguration{
RedirectAllRequestsTo: &s3.RedirectAllRequestsTo{HostName: "example.com"},
}
err := b.PutBucketWebsite(config)
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
body, err := ioutil.ReadAll(req.Body)
req.Body.Close()
c.Assert(err, check.IsNil)
c.Assert(string(body), check.Equals, BucketWebsiteConfigurationDump)
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/bucket/")
c.Assert(req.URL.RawQuery, check.Equals, "website=")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
}
// Head docs: http://bit.ly/17K1ylI
func (s *S) TestHead(c *check.C) {
testServer.Response(200, nil, "content")
b := s.s3.Bucket("bucket")
resp, err := b.Head("name", nil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "HEAD")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
c.Assert(err, check.IsNil)
c.Assert(resp.ContentLength, check.FitsTypeOf, int64(0))
c.Assert(resp, check.FitsTypeOf, &http.Response{})
}
// DeleteBucket docs: http://goo.gl/GoBrY
func (s *S) TestDelBucket(c *check.C) {
testServer.Response(204, nil, "")
b := s.s3.Bucket("bucket")
err := b.DelBucket()
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "DELETE")
c.Assert(req.URL.Path, check.Equals, "/bucket/")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
}
// GetObject docs: http://goo.gl/isCO7
func (s *S) TestGet(c *check.C) {
testServer.Response(200, nil, "content")
b := s.s3.Bucket("bucket")
data, err := b.Get("name")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
c.Assert(err, check.IsNil)
c.Assert(string(data), check.Equals, "content")
}
func (s *S) TestGetWithPlus(c *check.C) {
testServer.Response(200, nil, "content")
b := s.s3.Bucket("bucket")
_, err := b.Get("has+plus")
req := testServer.WaitRequest()
c.Assert(err, check.IsNil)
c.Assert(req.RequestURI, check.Equals, "http://localhost:4444/bucket/has%2Bplus")
}
func (s *S) TestURL(c *check.C) {
testServer.Response(200, nil, "content")
b := s.s3.Bucket("bucket")
url := b.URL("name")
r, err := http.Get(url)
c.Assert(err, check.IsNil)
data, err := ioutil.ReadAll(r.Body)
r.Body.Close()
c.Assert(err, check.IsNil)
c.Assert(string(data), check.Equals, "content")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
}
func (s *S) TestGetReader(c *check.C) {
testServer.Response(200, nil, "content")
b := s.s3.Bucket("bucket")
rc, err := b.GetReader("name")
c.Assert(err, check.IsNil)
data, err := ioutil.ReadAll(rc)
rc.Close()
c.Assert(err, check.IsNil)
c.Assert(string(data), check.Equals, "content")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
}
func (s *S) TestGetNotFound(c *check.C) {
for i := 0; i < 10; i++ {
testServer.Response(404, nil, GetObjectErrorDump)
}
b := s.s3.Bucket("non-existent-bucket")
data, err := b.Get("non-existent")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/non-existent-bucket/non-existent")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
s3err, _ := err.(*s3.Error)
c.Assert(s3err, check.NotNil)
c.Assert(s3err.StatusCode, check.Equals, 404)
c.Assert(s3err.BucketName, check.Equals, "non-existent-bucket")
c.Assert(s3err.RequestId, check.Equals, "3F1B667FAD71C3D8")
c.Assert(s3err.HostId, check.Equals, "L4ee/zrm1irFXY5F45fKXIRdOf9ktsKY/8TDVawuMK2jWRb1RF84i1uBzkdNqS5D")
c.Assert(s3err.Code, check.Equals, "NoSuchBucket")
c.Assert(s3err.Message, check.Equals, "The specified bucket does not exist")
c.Assert(s3err.Error(), check.Equals, "The specified bucket does not exist")
c.Assert(data, check.IsNil)
}
// PutObject docs: http://goo.gl/FEBPD
func (s *S) TestPutObject(c *check.C) {
testServer.Response(200, nil, "")
const DISPOSITION = "attachment; filename=\"0x1a2b3c.jpg\""
b := s.s3.Bucket("bucket")
err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{ContentDisposition: DISPOSITION})
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"})
c.Assert(req.Header["Content-Disposition"], check.DeepEquals, []string{DISPOSITION})
//c.Assert(req.Header["Content-MD5"], gocheck.DeepEquals, "...")
c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
}
func (s *S) TestPutObjectReducedRedundancy(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{StorageClass: s3.ReducedRedundancy})
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"})
c.Assert(req.Header["X-Amz-Storage-Class"], check.DeepEquals, []string{"REDUCED_REDUNDANCY"})
}
// PutCopy docs: http://goo.gl/mhEHtA
func (s *S) TestPutCopy(c *check.C) {
testServer.Response(200, nil, PutCopyResultDump)
b := s.s3.Bucket("bucket")
res, err := b.PutCopy("name", s3.Private, s3.CopyOptions{},
// 0xFC is &uuml; - 0xE9 is &eacute;
"source-bucket/\u00FCber-fil\u00E9.jpg")
c.Assert(err, check.IsNil)
c.Assert(res, check.DeepEquals, &s3.CopyObjectResult{
ETag: `"9b2cf535f27731c974343645a3985328"`,
LastModified: `2009-10-28T22:32:00`})
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"0"})
c.Assert(req.Header["X-Amz-Copy-Source"], check.DeepEquals, []string{`source-bucket%2F%C3%BCber-fil%C3%A9.jpg`})
c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
}
func (s *S) TestPutObjectReadTimeout(c *check.C) {
s.s3.ReadTimeout = 50 * time.Millisecond
defer func() {
s.s3.ReadTimeout = 0
}()
b := s.s3.Bucket("bucket")
err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{})
// Make sure that we get a timeout error.
c.Assert(err, check.NotNil)
// Set the response after the request times out so that the next request will work.
testServer.Response(200, nil, "")
// This time set the response within our timeout period so that we expect the call
// to return successfully.
go func() {
time.Sleep(25 * time.Millisecond)
testServer.Response(200, nil, "")
}()
err = b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
}
func (s *S) TestPutReader(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
buf := bytes.NewBufferString("content")
err := b.PutReader("name", buf, int64(buf.Len()), "content-type", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"})
//c.Assert(req.Header["Content-MD5"], gocheck.Equals, "...")
c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
}
// DelObject docs: http://goo.gl/APeTt
func (s *S) TestDelObject(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
err := b.Del("name")
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "DELETE")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
}
func (s *S) TestDelMultiObjects(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
objects := []s3.Object{s3.Object{Key: "test"}}
err := b.DelMulti(s3.Delete{
Quiet: false,
Objects: objects,
})
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "POST")
c.Assert(req.URL.RawQuery, check.Equals, "delete=")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
c.Assert(req.Header["Content-MD5"], check.Not(check.Equals), "")
c.Assert(req.Header["Content-Type"], check.Not(check.Equals), "")
c.Assert(req.ContentLength, check.Not(check.Equals), "")
}
// Bucket List Objects docs: http://goo.gl/YjQTc
func (s *S) TestList(c *check.C) {
testServer.Response(200, nil, GetListResultDump1)
b := s.s3.Bucket("quotes")
data, err := b.List("N", "", "", 0)
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/quotes/")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
c.Assert(req.Form["prefix"], check.DeepEquals, []string{"N"})
c.Assert(req.Form["delimiter"], check.DeepEquals, []string{""})
c.Assert(req.Form["marker"], check.DeepEquals, []string{""})
c.Assert(req.Form["max-keys"], check.DeepEquals, []string(nil))
c.Assert(data.Name, check.Equals, "quotes")
c.Assert(data.Prefix, check.Equals, "N")
c.Assert(data.IsTruncated, check.Equals, false)
c.Assert(len(data.Contents), check.Equals, 2)
c.Assert(data.Contents[0].Key, check.Equals, "Nelson")
c.Assert(data.Contents[0].LastModified, check.Equals, "2006-01-01T12:00:00.000Z")
c.Assert(data.Contents[0].ETag, check.Equals, `"828ef3fdfa96f00ad9f27c383fc9ac7f"`)
c.Assert(data.Contents[0].Size, check.Equals, int64(5))
c.Assert(data.Contents[0].StorageClass, check.Equals, "STANDARD")
c.Assert(data.Contents[0].Owner.ID, check.Equals, "bcaf161ca5fb16fd081034f")
c.Assert(data.Contents[0].Owner.DisplayName, check.Equals, "webfile")
c.Assert(data.Contents[1].Key, check.Equals, "Neo")
c.Assert(data.Contents[1].LastModified, check.Equals, "2006-01-01T12:00:00.000Z")
c.Assert(data.Contents[1].ETag, check.Equals, `"828ef3fdfa96f00ad9f27c383fc9ac7f"`)
c.Assert(data.Contents[1].Size, check.Equals, int64(4))
c.Assert(data.Contents[1].StorageClass, check.Equals, "STANDARD")
c.Assert(data.Contents[1].Owner.ID, check.Equals, "bcaf1ffd86a5fb16fd081034f")
c.Assert(data.Contents[1].Owner.DisplayName, check.Equals, "webfile")
}
func (s *S) TestListWithDelimiter(c *check.C) {
testServer.Response(200, nil, GetListResultDump2)
b := s.s3.Bucket("quotes")
data, err := b.List("photos/2006/", "/", "some-marker", 1000)
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "GET")
c.Assert(req.URL.Path, check.Equals, "/quotes/")
c.Assert(req.Header["Date"], check.Not(check.Equals), "")
c.Assert(req.Form["prefix"], check.DeepEquals, []string{"photos/2006/"})
c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"})
c.Assert(req.Form["marker"], check.DeepEquals, []string{"some-marker"})
c.Assert(req.Form["max-keys"], check.DeepEquals, []string{"1000"})
c.Assert(data.Name, check.Equals, "example-bucket")
c.Assert(data.Prefix, check.Equals, "photos/2006/")
c.Assert(data.Delimiter, check.Equals, "/")
c.Assert(data.Marker, check.Equals, "some-marker")
c.Assert(data.IsTruncated, check.Equals, false)
c.Assert(len(data.Contents), check.Equals, 0)
c.Assert(data.CommonPrefixes, check.DeepEquals, []string{"photos/2006/feb/", "photos/2006/jan/"})
}
func (s *S) TestExists(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
result, err := b.Exists("name")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "HEAD")
c.Assert(err, check.IsNil)
c.Assert(result, check.Equals, true)
}
func (s *S) TestExistsNotFound404(c *check.C) {
testServer.Response(404, nil, "")
b := s.s3.Bucket("bucket")
result, err := b.Exists("name")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "HEAD")
c.Assert(err, check.IsNil)
c.Assert(result, check.Equals, false)
}
func (s *S) TestExistsNotFound403(c *check.C) {
testServer.Response(403, nil, "")
b := s.s3.Bucket("bucket")
result, err := b.Exists("name")
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "HEAD")
c.Assert(err, check.IsNil)
c.Assert(result, check.Equals, false)
}
func (s *S) TestGetService(c *check.C) {
testServer.Response(200, nil, GetServiceDump)
expected := s3.GetServiceResp{
Owner: s3.Owner{
ID: "bcaf1ffd86f461ca5fb16fd081034f",
DisplayName: "webfile",
},
Buckets: []s3.BucketInfo{
s3.BucketInfo{
Name: "quotes",
CreationDate: "2006-02-03T16:45:09.000Z",
},
s3.BucketInfo{
Name: "samples",
CreationDate: "2006-02-03T16:41:58.000Z",
},
},
}
received, err := s.s3.GetService()
c.Assert(err, check.IsNil)
c.Assert(*received, check.DeepEquals, expected)
}
func (s *S) TestLocation(c *check.C) {
testServer.Response(200, nil, GetLocationUsStandard)
expectedUsStandard := "us-east-1"
bucketUsStandard := s.s3.Bucket("us-east-1")
resultUsStandard, err := bucketUsStandard.Location()
c.Assert(err, check.IsNil)
c.Assert(resultUsStandard, check.Equals, expectedUsStandard)
testServer.Response(200, nil, GetLocationUsWest1)
expectedUsWest1 := "us-west-1"
bucketUsWest1 := s.s3.Bucket("us-west-1")
resultUsWest1, err := bucketUsWest1.Location()
c.Assert(err, check.IsNil)
c.Assert(resultUsWest1, check.Equals, expectedUsWest1)
}
func (s *S) TestSupportRadosGW(c *check.C) {
testServer.Response(200, nil, "content")
s.s3.Region.Name = "generic"
b := s.s3.Bucket("bucket")
_, err := b.Get("rgw")
req := testServer.WaitRequest()
c.Assert(err, check.IsNil)
c.Assert(req.RequestURI, check.Equals, "/bucket/rgw")
}

View file

@ -1,603 +0,0 @@
package s3_test
import (
"bytes"
"crypto/md5"
"fmt"
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"github.com/AdRoll/goamz/testutil"
"gopkg.in/check.v1"
"io/ioutil"
"net"
"net/http"
"sort"
"strings"
"time"
)
// AmazonServer represents an Amazon S3 server.
type AmazonServer struct {
auth aws.Auth
}
func (s *AmazonServer) SetUp(c *check.C) {
auth, err := aws.EnvAuth()
if err != nil {
c.Fatal(err.Error())
}
s.auth = auth
}
var _ = check.Suite(&AmazonClientSuite{Region: aws.USEast})
var _ = check.Suite(&AmazonClientSuite{Region: aws.EUWest})
var _ = check.Suite(&AmazonDomainClientSuite{Region: aws.USEast})
// AmazonClientSuite tests the client against a live S3 server.
type AmazonClientSuite struct {
aws.Region
srv AmazonServer
ClientTests
}
func (s *AmazonClientSuite) SetUpSuite(c *check.C) {
if !testutil.Amazon {
c.Skip("live tests against AWS disabled (no -amazon)")
}
s.srv.SetUp(c)
s.s3 = s3.New(s.srv.auth, s.Region)
// In case tests were interrupted in the middle before.
s.ClientTests.Cleanup()
}
func (s *AmazonClientSuite) TearDownTest(c *check.C) {
s.ClientTests.Cleanup()
}
// AmazonDomainClientSuite tests the client against a live S3
// server using bucket names in the endpoint domain name rather
// than the request path.
type AmazonDomainClientSuite struct {
aws.Region
srv AmazonServer
ClientTests
}
func (s *AmazonDomainClientSuite) SetUpSuite(c *check.C) {
if !testutil.Amazon {
c.Skip("live tests against AWS disabled (no -amazon)")
}
s.srv.SetUp(c)
region := s.Region
region.S3BucketEndpoint = "https://${bucket}.s3.amazonaws.com"
s.s3 = s3.New(s.srv.auth, region)
s.ClientTests.Cleanup()
}
func (s *AmazonDomainClientSuite) TearDownTest(c *check.C) {
s.ClientTests.Cleanup()
}
// ClientTests defines integration tests designed to test the client.
// It is not used as a test suite in itself, but embedded within
// another type.
type ClientTests struct {
s3 *s3.S3
authIsBroken bool
}
func (s *ClientTests) Cleanup() {
killBucket(testBucket(s.s3))
}
func testBucket(s *s3.S3) *s3.Bucket {
// Watch out! If this function is corrupted and made to match with something
// people own, killBucket will happily remove *everything* inside the bucket.
key := s.Auth.AccessKey
if len(key) >= 8 {
key = s.Auth.AccessKey[:8]
}
return s.Bucket(fmt.Sprintf("goamz-%s-%s", s.Region.Name, key))
}
var attempts = aws.AttemptStrategy{
Min: 5,
Total: 20 * time.Second,
Delay: 100 * time.Millisecond,
}
func killBucket(b *s3.Bucket) {
var err error
for attempt := attempts.Start(); attempt.Next(); {
err = b.DelBucket()
if err == nil {
return
}
if _, ok := err.(*net.DNSError); ok {
return
}
e, ok := err.(*s3.Error)
if ok && e.Code == "NoSuchBucket" {
return
}
if ok && e.Code == "BucketNotEmpty" {
// Errors are ignored here. Just retry.
resp, err := b.List("", "", "", 1000)
if err == nil {
for _, key := range resp.Contents {
_ = b.Del(key.Key)
}
}
multis, _, _ := b.ListMulti("", "")
for _, m := range multis {
_ = m.Abort()
}
}
}
message := "cannot delete test bucket"
if err != nil {
message += ": " + err.Error()
}
panic(message)
}
func get(url string) ([]byte, error) {
for attempt := attempts.Start(); attempt.Next(); {
resp, err := http.Get(url)
if err != nil {
if attempt.HasNext() {
continue
}
return nil, err
}
data, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
if attempt.HasNext() {
continue
}
return nil, err
}
return data, err
}
panic("unreachable")
}
func (s *ClientTests) TestBasicFunctionality(c *check.C) {
b := testBucket(s.s3)
err := b.PutBucket(s3.PublicRead)
c.Assert(err, check.IsNil)
err = b.Put("name", []byte("yo!"), "text/plain", s3.PublicRead, s3.Options{})
c.Assert(err, check.IsNil)
defer b.Del("name")
data, err := b.Get("name")
c.Assert(err, check.IsNil)
c.Assert(string(data), check.Equals, "yo!")
data, err = get(b.URL("name"))
c.Assert(err, check.IsNil)
c.Assert(string(data), check.Equals, "yo!")
buf := bytes.NewBufferString("hey!")
err = b.PutReader("name2", buf, int64(buf.Len()), "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
defer b.Del("name2")
rc, err := b.GetReader("name2")
c.Assert(err, check.IsNil)
data, err = ioutil.ReadAll(rc)
c.Check(err, check.IsNil)
c.Check(string(data), check.Equals, "hey!")
rc.Close()
data, err = get(b.SignedURL("name2", time.Now().Add(time.Hour)))
c.Assert(err, check.IsNil)
c.Assert(string(data), check.Equals, "hey!")
if !s.authIsBroken {
data, err = get(b.SignedURL("name2", time.Now().Add(-time.Hour)))
c.Assert(err, check.IsNil)
c.Assert(string(data), check.Matches, "(?s).*AccessDenied.*")
}
err = b.DelBucket()
c.Assert(err, check.NotNil)
s3err, ok := err.(*s3.Error)
c.Assert(ok, check.Equals, true)
c.Assert(s3err.Code, check.Equals, "BucketNotEmpty")
c.Assert(s3err.BucketName, check.Equals, b.Name)
c.Assert(s3err.Message, check.Equals, "The bucket you tried to delete is not empty")
err = b.Del("name")
c.Assert(err, check.IsNil)
err = b.Del("name2")
c.Assert(err, check.IsNil)
err = b.DelBucket()
c.Assert(err, check.IsNil)
}
func (s *ClientTests) TestGetNotFound(c *check.C) {
b := s.s3.Bucket("goamz-" + s.s3.Auth.AccessKey)
data, err := b.Get("non-existent")
s3err, _ := err.(*s3.Error)
c.Assert(s3err, check.NotNil)
c.Assert(s3err.StatusCode, check.Equals, 404)
c.Assert(s3err.Code, check.Equals, "NoSuchBucket")
c.Assert(s3err.Message, check.Equals, "The specified bucket does not exist")
c.Assert(data, check.IsNil)
}
// Communicate with all endpoints to see if they are alive.
func (s *ClientTests) TestRegions(c *check.C) {
errs := make(chan error, len(aws.Regions))
for _, region := range aws.Regions {
go func(r aws.Region) {
s := s3.New(s.s3.Auth, r)
b := s.Bucket("goamz-" + s.Auth.AccessKey)
_, err := b.Get("non-existent")
errs <- err
}(region)
}
for _ = range aws.Regions {
err := <-errs
if err != nil {
s3_err, ok := err.(*s3.Error)
if ok {
c.Check(s3_err.Code, check.Matches, "NoSuchBucket")
} else if _, ok = err.(*net.DNSError); ok {
// Okay as well.
} else {
c.Errorf("Non-S3 error: %s", err)
}
} else {
c.Errorf("Test should have errored but it seems to have succeeded")
}
}
}
var objectNames = []string{
"index.html",
"index2.html",
"photos/2006/February/sample2.jpg",
"photos/2006/February/sample3.jpg",
"photos/2006/February/sample4.jpg",
"photos/2006/January/sample.jpg",
"test/bar",
"test/foo",
}
func keys(names ...string) []s3.Key {
ks := make([]s3.Key, len(names))
for i, name := range names {
ks[i].Key = name
}
return ks
}
// As the ListResp specifies all the parameters to the
// request too, we use it to specify request parameters
// and expected results. The Contents field is
// used only for the key names inside it.
var listTests = []s3.ListResp{
// normal list.
{
Contents: keys(objectNames...),
}, {
Marker: objectNames[0],
Contents: keys(objectNames[1:]...),
}, {
Marker: objectNames[0] + "a",
Contents: keys(objectNames[1:]...),
}, {
Marker: "z",
},
// limited results.
{
MaxKeys: 2,
Contents: keys(objectNames[0:2]...),
IsTruncated: true,
}, {
MaxKeys: 2,
Marker: objectNames[0],
Contents: keys(objectNames[1:3]...),
IsTruncated: true,
}, {
MaxKeys: 2,
Marker: objectNames[len(objectNames)-2],
Contents: keys(objectNames[len(objectNames)-1:]...),
},
// with delimiter
{
Delimiter: "/",
CommonPrefixes: []string{"photos/", "test/"},
Contents: keys("index.html", "index2.html"),
}, {
Delimiter: "/",
Prefix: "photos/2006/",
CommonPrefixes: []string{"photos/2006/February/", "photos/2006/January/"},
}, {
Delimiter: "/",
Prefix: "t",
CommonPrefixes: []string{"test/"},
}, {
Delimiter: "/",
MaxKeys: 1,
Contents: keys("index.html"),
IsTruncated: true,
}, {
Delimiter: "/",
MaxKeys: 1,
Marker: "index2.html",
CommonPrefixes: []string{"photos/"},
IsTruncated: true,
}, {
Delimiter: "/",
MaxKeys: 1,
Marker: "photos/",
CommonPrefixes: []string{"test/"},
IsTruncated: false,
}, {
Delimiter: "Feb",
CommonPrefixes: []string{"photos/2006/Feb"},
Contents: keys("index.html", "index2.html", "photos/2006/January/sample.jpg", "test/bar", "test/foo"),
},
}
func (s *ClientTests) TestDoublePutBucket(c *check.C) {
b := testBucket(s.s3)
err := b.PutBucket(s3.PublicRead)
c.Assert(err, check.IsNil)
err = b.PutBucket(s3.PublicRead)
if err != nil {
c.Assert(err, check.FitsTypeOf, new(s3.Error))
c.Assert(err.(*s3.Error).Code, check.Equals, "BucketAlreadyOwnedByYou")
}
}
func (s *ClientTests) TestBucketList(c *check.C) {
b := testBucket(s.s3)
err := b.PutBucket(s3.Private)
c.Assert(err, check.IsNil)
objData := make(map[string][]byte)
for i, path := range objectNames {
data := []byte(strings.Repeat("a", i))
err := b.Put(path, data, "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
defer b.Del(path)
objData[path] = data
}
for i, t := range listTests {
c.Logf("test %d", i)
resp, err := b.List(t.Prefix, t.Delimiter, t.Marker, t.MaxKeys)
c.Assert(err, check.IsNil)
c.Check(resp.Name, check.Equals, b.Name)
c.Check(resp.Delimiter, check.Equals, t.Delimiter)
c.Check(resp.IsTruncated, check.Equals, t.IsTruncated)
c.Check(resp.CommonPrefixes, check.DeepEquals, t.CommonPrefixes)
checkContents(c, resp.Contents, objData, t.Contents)
}
}
func etag(data []byte) string {
sum := md5.New()
sum.Write(data)
return fmt.Sprintf(`"%x"`, sum.Sum(nil))
}
func checkContents(c *check.C, contents []s3.Key, data map[string][]byte, expected []s3.Key) {
c.Assert(contents, check.HasLen, len(expected))
for i, k := range contents {
c.Check(k.Key, check.Equals, expected[i].Key)
// TODO mtime
c.Check(k.Size, check.Equals, int64(len(data[k.Key])))
c.Check(k.ETag, check.Equals, etag(data[k.Key]))
}
}
func (s *ClientTests) TestMultiInitPutList(c *check.C) {
b := testBucket(s.s3)
err := b.PutBucket(s3.Private)
c.Assert(err, check.IsNil)
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
c.Assert(multi.UploadId, check.Matches, ".+")
defer multi.Abort()
var sent []s3.Part
for i := 0; i < 5; i++ {
p, err := multi.PutPart(i+1, strings.NewReader(fmt.Sprintf("<part %d>", i+1)))
c.Assert(err, check.IsNil)
c.Assert(p.N, check.Equals, i+1)
c.Assert(p.Size, check.Equals, int64(8))
c.Assert(p.ETag, check.Matches, ".+")
sent = append(sent, p)
}
s3.SetListPartsMax(2)
parts, err := multi.ListParts()
c.Assert(err, check.IsNil)
c.Assert(parts, check.HasLen, len(sent))
for i := range parts {
c.Assert(parts[i].N, check.Equals, sent[i].N)
c.Assert(parts[i].Size, check.Equals, sent[i].Size)
c.Assert(parts[i].ETag, check.Equals, sent[i].ETag)
}
err = multi.Complete(parts)
s3err, failed := err.(*s3.Error)
c.Assert(failed, check.Equals, true)
c.Assert(s3err.Code, check.Equals, "EntityTooSmall")
err = multi.Abort()
c.Assert(err, check.IsNil)
_, err = multi.ListParts()
s3err, ok := err.(*s3.Error)
c.Assert(ok, check.Equals, true)
c.Assert(s3err.Code, check.Equals, "NoSuchUpload")
}
// This may take a minute or more due to the minimum size accepted S3
// on multipart upload parts.
func (s *ClientTests) TestMultiComplete(c *check.C) {
b := testBucket(s.s3)
err := b.PutBucket(s3.Private)
c.Assert(err, check.IsNil)
contentType := "text/plain"
meta := make(map[string][]string)
meta["X-Amz-Meta-TestField"] = []string{"testValue"}
options := s3.Options{ContentEncoding: "identity", ContentDisposition: "inline", Meta: meta}
multi, err := b.InitMulti("multi", contentType, s3.Private, options)
c.Assert(err, check.IsNil)
c.Assert(multi.UploadId, check.Matches, ".+")
defer multi.Abort()
// Minimum size S3 accepts for all but the last part is 5MB.
data1 := make([]byte, 5*1024*1024)
data2 := []byte("<part 2>")
part1, err := multi.PutPart(1, bytes.NewReader(data1))
c.Assert(err, check.IsNil)
part2, err := multi.PutPart(2, bytes.NewReader(data2))
c.Assert(err, check.IsNil)
// Purposefully reversed. The order requirement must be handled.
err = multi.Complete([]s3.Part{part2, part1})
c.Assert(err, check.IsNil)
data, err := b.Get("multi")
c.Assert(err, check.IsNil)
c.Assert(len(data), check.Equals, len(data1)+len(data2))
for i := range data1 {
if data[i] != data1[i] {
c.Fatalf("uploaded object at byte %d: want %d, got %d", data1[i], data[i])
}
}
c.Assert(string(data[len(data1):]), check.Equals, string(data2))
resp, err := b.GetResponse("multi")
c.Assert(resp.Header.Get("Content-Type"), check.Equals, contentType)
c.Assert(resp.Header.Get("x-amz-acl"), check.Equals, s3.Private)
c.Assert(resp.Header.Get("Content-MD5"), check.Equals, options.ContentMD5)
c.Assert(resp.Header.Get("Content-Encoding"), check.Equals, options.ContentEncoding)
c.Assert(resp.Header.Get("Content-Disposition"), check.Equals, options.ContentDisposition)
for k, values := range meta {
c.Assert(resp.Header.Get(k), check.Equals, strings.Join(values, ","))
}
}
type multiList []*s3.Multi
func (l multiList) Len() int { return len(l) }
func (l multiList) Less(i, j int) bool { return l[i].Key < l[j].Key }
func (l multiList) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
func (s *ClientTests) TestListMulti(c *check.C) {
b := testBucket(s.s3)
err := b.PutBucket(s3.Private)
c.Assert(err, check.IsNil)
// Ensure an empty state before testing its behavior.
multis, _, err := b.ListMulti("", "")
for _, m := range multis {
err := m.Abort()
c.Assert(err, check.IsNil)
}
keys := []string{
"a/multi2",
"a/multi3",
"b/multi4",
"multi1",
}
for _, key := range keys {
m, err := b.InitMulti(key, "", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
defer m.Abort()
}
// Amazon's implementation of the multiple-request listing for
// multipart uploads in progress seems broken in multiple ways.
// (next tokens are not provided, etc).
//s3.SetListMultiMax(2)
multis, prefixes, err := b.ListMulti("", "")
c.Assert(err, check.IsNil)
for attempt := attempts.Start(); attempt.Next() && len(multis) < len(keys); {
multis, prefixes, err = b.ListMulti("", "")
c.Assert(err, check.IsNil)
}
sort.Sort(multiList(multis))
c.Assert(prefixes, check.IsNil)
var gotKeys []string
for _, m := range multis {
gotKeys = append(gotKeys, m.Key)
}
c.Assert(gotKeys, check.DeepEquals, keys)
for _, m := range multis {
c.Assert(m.Bucket, check.Equals, b)
c.Assert(m.UploadId, check.Matches, ".+")
}
multis, prefixes, err = b.ListMulti("", "/")
for attempt := attempts.Start(); attempt.Next() && len(prefixes) < 2; {
multis, prefixes, err = b.ListMulti("", "")
c.Assert(err, check.IsNil)
}
c.Assert(err, check.IsNil)
c.Assert(prefixes, check.DeepEquals, []string{"a/", "b/"})
c.Assert(multis, check.HasLen, 1)
c.Assert(multis[0].Bucket, check.Equals, b)
c.Assert(multis[0].Key, check.Equals, "multi1")
c.Assert(multis[0].UploadId, check.Matches, ".+")
for attempt := attempts.Start(); attempt.Next() && len(multis) < 2; {
multis, prefixes, err = b.ListMulti("", "")
c.Assert(err, check.IsNil)
}
multis, prefixes, err = b.ListMulti("a/", "/")
c.Assert(err, check.IsNil)
c.Assert(prefixes, check.IsNil)
c.Assert(multis, check.HasLen, 2)
c.Assert(multis[0].Bucket, check.Equals, b)
c.Assert(multis[0].Key, check.Equals, "a/multi2")
c.Assert(multis[0].UploadId, check.Matches, ".+")
c.Assert(multis[1].Bucket, check.Equals, b)
c.Assert(multis[1].Key, check.Equals, "a/multi3")
c.Assert(multis[1].UploadId, check.Matches, ".+")
}
func (s *ClientTests) TestMultiPutAllZeroLength(c *check.C) {
b := testBucket(s.s3)
err := b.PutBucket(s3.Private)
c.Assert(err, check.IsNil)
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
defer multi.Abort()
// This tests an edge case. Amazon requires at least one
// part for multiprat uploads to work, even the part is empty.
parts, err := multi.PutAll(strings.NewReader(""), 5*1024*1024)
c.Assert(err, check.IsNil)
c.Assert(parts, check.HasLen, 1)
c.Assert(parts[0].Size, check.Equals, int64(0))
c.Assert(parts[0].ETag, check.Equals, `"d41d8cd98f00b204e9800998ecf8427e"`)
err = multi.Complete(parts)
c.Assert(err, check.IsNil)
}

View file

@ -1,87 +0,0 @@
package s3_test
import (
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"github.com/AdRoll/goamz/s3/s3test"
"github.com/AdRoll/goamz/testutil"
"gopkg.in/check.v1"
)
type LocalServer struct {
auth aws.Auth
region aws.Region
srv *s3test.Server
config *s3test.Config
}
func (s *LocalServer) SetUp(c *check.C) {
srv, err := s3test.NewServer(s.config)
c.Assert(err, check.IsNil)
c.Assert(srv, check.NotNil)
s.srv = srv
s.region = aws.Region{
Name: "faux-region-1",
S3Endpoint: srv.URL(),
S3LocationConstraint: true, // s3test server requires a LocationConstraint
}
}
// LocalServerSuite defines tests that will run
// against the local s3test server. It includes
// selected tests from ClientTests;
// when the s3test functionality is sufficient, it should
// include all of them, and ClientTests can be simply embedded.
type LocalServerSuite struct {
srv LocalServer
clientTests ClientTests
}
var (
// run tests twice, once in us-east-1 mode, once not.
_ = check.Suite(&LocalServerSuite{})
_ = check.Suite(&LocalServerSuite{
srv: LocalServer{
config: &s3test.Config{
Send409Conflict: true,
},
},
})
)
func (s *LocalServerSuite) SetUpSuite(c *check.C) {
s.srv.SetUp(c)
s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region)
// TODO Sadly the fake server ignores auth completely right now. :-(
s.clientTests.authIsBroken = true
s.clientTests.Cleanup()
}
func (s *LocalServerSuite) TearDownTest(c *check.C) {
s.clientTests.Cleanup()
}
func (s *LocalServerSuite) TestBasicFunctionality(c *check.C) {
s.clientTests.TestBasicFunctionality(c)
}
func (s *LocalServerSuite) TestGetNotFound(c *check.C) {
s.clientTests.TestGetNotFound(c)
}
func (s *LocalServerSuite) TestBucketList(c *check.C) {
s.clientTests.TestBucketList(c)
}
func (s *LocalServerSuite) TestDoublePutBucket(c *check.C) {
s.clientTests.TestDoublePutBucket(c)
}
func (s *LocalServerSuite) TestMultiComplete(c *check.C) {
if !testutil.Amazon {
c.Skip("live tests against AWS disabled (no -amazon)")
}
s.clientTests.TestMultiComplete(c)
}

View file

@ -1,148 +0,0 @@
package s3_test
import (
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"gopkg.in/check.v1"
)
// S3 ReST authentication docs: http://goo.gl/G1LrK
var testAuth = aws.Auth{AccessKey: "0PN5J17HBGZHT7JJ3X82", SecretKey: "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"}
func (s *S) TestSignExampleObjectGet(c *check.C) {
method := "GET"
path := "/johnsmith/photos/puppy.jpg"
headers := map[string][]string{
"Host": {"johnsmith.s3.amazonaws.com"},
"Date": {"Tue, 27 Mar 2007 19:36:42 +0000"},
}
s3.Sign(testAuth, method, path, nil, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}
func (s *S) TestSignExampleObjectPut(c *check.C) {
method := "PUT"
path := "/johnsmith/photos/puppy.jpg"
headers := map[string][]string{
"Host": {"johnsmith.s3.amazonaws.com"},
"Date": {"Tue, 27 Mar 2007 21:15:45 +0000"},
"Content-Type": {"image/jpeg"},
"Content-Length": {"94328"},
}
s3.Sign(testAuth, method, path, nil, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}
func (s *S) TestSignExampleList(c *check.C) {
method := "GET"
path := "/johnsmith/"
params := map[string][]string{
"prefix": {"photos"},
"max-keys": {"50"},
"marker": {"puppy"},
}
headers := map[string][]string{
"Host": {"johnsmith.s3.amazonaws.com"},
"Date": {"Tue, 27 Mar 2007 19:42:41 +0000"},
"User-Agent": {"Mozilla/5.0"},
}
s3.Sign(testAuth, method, path, params, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}
func (s *S) TestSignExampleFetch(c *check.C) {
method := "GET"
path := "/johnsmith/"
params := map[string][]string{
"acl": {""},
}
headers := map[string][]string{
"Host": {"johnsmith.s3.amazonaws.com"},
"Date": {"Tue, 27 Mar 2007 19:44:46 +0000"},
}
s3.Sign(testAuth, method, path, params, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}
func (s *S) TestSignExampleDelete(c *check.C) {
method := "DELETE"
path := "/johnsmith/photos/puppy.jpg"
params := map[string][]string{}
headers := map[string][]string{
"Host": {"s3.amazonaws.com"},
"Date": {"Tue, 27 Mar 2007 21:20:27 +0000"},
"User-Agent": {"dotnet"},
"x-amz-date": {"Tue, 27 Mar 2007 21:20:26 +0000"},
}
s3.Sign(testAuth, method, path, params, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}
func (s *S) TestSignExampleUpload(c *check.C) {
method := "PUT"
path := "/static.johnsmith.net/db-backup.dat.gz"
params := map[string][]string{}
headers := map[string][]string{
"Host": {"static.johnsmith.net:8080"},
"Date": {"Tue, 27 Mar 2007 21:06:08 +0000"},
"User-Agent": {"curl/7.15.5"},
"x-amz-acl": {"public-read"},
"content-type": {"application/x-download"},
"Content-MD5": {"4gJE4saaMU4BqNR0kLY+lw=="},
"X-Amz-Meta-ReviewedBy": {"joe@johnsmith.net,jane@johnsmith.net"},
"X-Amz-Meta-FileChecksum": {"0x02661779"},
"X-Amz-Meta-ChecksumAlgorithm": {"crc32"},
"Content-Disposition": {"attachment; filename=database.dat"},
"Content-Encoding": {"gzip"},
"Content-Length": {"5913339"},
}
s3.Sign(testAuth, method, path, params, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}
func (s *S) TestSignExampleListAllMyBuckets(c *check.C) {
method := "GET"
path := "/"
headers := map[string][]string{
"Host": {"s3.amazonaws.com"},
"Date": {"Wed, 28 Mar 2007 01:29:59 +0000"},
}
s3.Sign(testAuth, method, path, nil, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:Db+gepJSUbZKwpx1FR0DLtEYoZA="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}
func (s *S) TestSignExampleUnicodeKeys(c *check.C) {
method := "GET"
path := "/dictionary/fran%C3%A7ais/pr%c3%a9f%c3%a8re"
headers := map[string][]string{
"Host": {"s3.amazonaws.com"},
"Date": {"Wed, 28 Mar 2007 01:49:49 +0000"},
}
s3.Sign(testAuth, method, path, nil, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:dxhSBHoI6eVSPcXJqEghlUzZMnY="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}
func (s *S) TestSignExampleCustomSSE(c *check.C) {
method := "GET"
path := "/secret/config"
params := map[string][]string{}
headers := map[string][]string{
"Host": {"secret.johnsmith.net:8080"},
"Date": {"Tue, 27 Mar 2007 21:06:08 +0000"},
"x-amz-server-side-encryption-customer-key": {"MWJhakVna1dQT1B0SDFMeGtVVnRQRTFGaU1ldFJrU0I="},
"x-amz-server-side-encryption-customer-key-MD5": {"glIqxpqQ4a9aoK/iLttKzQ=="},
"x-amz-server-side-encryption-customer-algorithm": {"AES256"},
}
s3.Sign(testAuth, method, path, params, headers)
expected := "AWS 0PN5J17HBGZHT7JJ3X82:Xq6PWmIo0aOWq+LDjCEiCGgbmHE="
c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
}

View file

@ -307,20 +307,42 @@ func (s *V4Signer) canonicalURI(u *url.URL) string {
}
func (s *V4Signer) canonicalQueryString(u *url.URL) string {
var a []string
keyValues := make(map[string]string, len(u.Query()))
keys := make([]string, len(u.Query()))
key_i := 0
for k, vs := range u.Query() {
k = url.QueryEscape(k)
for _, v := range vs {
if v == "" {
a = append(a, k+"=")
} else {
v = url.QueryEscape(v)
a = append(a, k+"="+v)
}
a := make([]string, len(vs))
for idx, v := range vs {
v = url.QueryEscape(v)
a[idx] = fmt.Sprintf("%s=%s", k, v)
}
keyValues[k] = strings.Join(a, "&")
keys[key_i] = k
key_i++
}
sort.Strings(a)
return strings.Join(a, "&")
sort.Strings(keys)
query := make([]string, len(keys))
for idx, key := range keys {
query[idx] = keyValues[key]
}
query_str := strings.Join(query, "&")
// AWS V4 signing requires that the space characters
// are encoded as %20 instead of +. On the other hand
// golangs url.QueryEscape as well as url.Values.Encode()
// both encode the space as a + character. See:
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
// https://github.com/golang/go/issues/4013
// https://groups.google.com/forum/#!topic/golang-nuts/BB443qEjPIk
return strings.Replace(query_str, "+", "%20", -1)
}
func (s *V4Signer) canonicalHeaders(h http.Header) string {

View file

@ -7,7 +7,7 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"github.com/AdRoll/goamz/aws"
"github.com/docker/goamz/aws"
"net/url"
"strconv"
"strings"

View file

@ -30,7 +30,7 @@ import (
"strings"
"time"
"github.com/AdRoll/goamz/aws"
"github.com/docker/goamz/aws"
)
const debug = false
@ -39,10 +39,9 @@ const debug = false
type S3 struct {
aws.Auth
aws.Region
ConnectTimeout time.Duration
ReadTimeout time.Duration
Signature int
private byte // Reserve the right of using private data.
Signature int
Client *http.Client
private byte // Reserve the right of using private data.
}
// The Bucket type encapsulates operations with an S3 bucket.
@ -61,6 +60,8 @@ type Owner struct {
//
type Options struct {
SSE bool
SSEKMS bool
SSEKMSKeyId string
SSECustomerAlgorithm string
SSECustomerKey string
SSECustomerKeyMD5 string
@ -96,7 +97,13 @@ var attempts = aws.AttemptStrategy{
// New creates a new S3.
func New(auth aws.Auth, region aws.Region) *S3 {
return &S3{auth, region, 0, 0, aws.V2Signature, 0}
return &S3{
Auth: auth,
Region: region,
Signature: aws.V2Signature,
Client: http.DefaultClient,
private: 0,
}
}
// Bucket returns a Bucket with the given name.
@ -171,6 +178,13 @@ const (
StandardStorage = StorageClass("STANDARD")
)
type ServerSideEncryption string
const (
S3Managed = ServerSideEncryption("AES256")
KMSManaged = ServerSideEncryption("aws:kms")
)
// PutBucket creates a new bucket.
//
// See http://goo.gl/ndjnR for details.
@ -344,7 +358,7 @@ func (b *Bucket) Put(path string, data []byte, contType string, perm ACL, option
func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (*CopyObjectResult, error) {
headers := map[string][]string{
"x-amz-acl": {string(perm)},
"x-amz-copy-source": {url.QueryEscape(source)},
"x-amz-copy-source": {escapePath(source)},
}
options.addHeaders(headers)
req := &request{
@ -383,7 +397,12 @@ func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType stri
// addHeaders adds o's specified fields to headers
func (o Options) addHeaders(headers map[string][]string) {
if o.SSE {
headers["x-amz-server-side-encryption"] = []string{"AES256"}
headers["x-amz-server-side-encryption"] = []string{string(S3Managed)}
} else if o.SSEKMS {
headers["x-amz-server-side-encryption"] = []string{string(KMSManaged)}
if len(o.SSEKMSKeyId) != 0 {
headers["x-amz-server-side-encryption-aws-kms-key-id"] = []string{o.SSEKMSKeyId}
}
} else if len(o.SSECustomerAlgorithm) != 0 && len(o.SSECustomerKey) != 0 && len(o.SSECustomerKeyMD5) != 0 {
// Amazon-managed keys and customer-managed keys are mutually exclusive
headers["x-amz-server-side-encryption-customer-algorithm"] = []string{o.SSECustomerAlgorithm}
@ -886,6 +905,12 @@ func (b *Bucket) PostFormArgsEx(path string, expires time.Time, redirect string,
"key": path,
}
if token := b.S3.Auth.Token(); token != "" {
fields["x-amz-security-token"] = token
conditions = append(conditions,
fmt.Sprintf("{\"x-amz-security-token\": \"%s\"}", token))
}
if conds != nil {
conditions = append(conditions, conds...)
}
@ -1123,7 +1148,6 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
Method: req.method,
ProtoMajor: 1,
ProtoMinor: 1,
Close: true,
Header: req.headers,
Form: req.params,
}
@ -1143,28 +1167,7 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
// If resp is not nil, the XML data contained in the response
// body will be unmarshalled on it.
func (s3 *S3) doHttpRequest(hreq *http.Request, resp interface{}) (*http.Response, error) {
c := http.Client{
Transport: &http.Transport{
Dial: func(netw, addr string) (c net.Conn, err error) {
deadline := time.Now().Add(s3.ReadTimeout)
if s3.ConnectTimeout > 0 {
c, err = net.DialTimeout(netw, addr, s3.ConnectTimeout)
} else {
c, err = net.Dial(netw, addr)
}
if err != nil {
return
}
if s3.ReadTimeout > 0 {
err = c.SetDeadline(deadline)
}
return
},
Proxy: http.ProxyFromEnvironment,
},
}
hresp, err := c.Do(hreq)
hresp, err := s3.Client.Do(hreq)
if err != nil {
return nil, err
}
@ -1296,3 +1299,7 @@ func hasCode(err error, code string) bool {
s3err, ok := err.(*Error)
return ok && s3err.Code == code
}
func escapePath(s string) string {
return (&url.URL{Path: s}).String()
}

View file

@ -7,7 +7,7 @@ import (
"encoding/hex"
"encoding/xml"
"fmt"
"github.com/AdRoll/goamz/s3"
"github.com/docker/goamz/s3"
"io"
"io/ioutil"
"log"
@ -44,6 +44,17 @@ type action struct {
reqId string
}
// A Clock reports the current time.
type Clock interface {
Now() time.Time
}
type realClock struct{}
func (c *realClock) Now() time.Time {
return time.Now()
}
// Config controls the internal behaviour of the Server. A nil config is the default
// and behaves as if all configurations assume their default behaviour. Once passed
// to NewServer, the configuration must not be modified.
@ -58,6 +69,10 @@ type Config struct {
// Address on which to listen. By default, a random port is assigned by the
// operating system and the server listens on localhost.
ListenAddress string
// Clock used to set mtime when updating an object. If nil,
// use the real clock.
Clock Clock
}
func (c *Config) send409Conflict() bool {
@ -76,6 +91,7 @@ type Server struct {
mu sync.Mutex
buckets map[string]*bucket
config *Config
closed bool
}
type bucket struct {
@ -129,10 +145,18 @@ type resource interface {
func NewServer(config *Config) (*Server, error) {
listenAddress := "localhost:0"
if config != nil && config.ListenAddress != "" {
if config == nil {
config = &Config{}
}
if config.ListenAddress != "" {
listenAddress = config.ListenAddress
}
if config.Clock == nil {
config.Clock = &realClock{}
}
l, err := net.Listen("tcp", listenAddress)
if err != nil {
return nil, fmt.Errorf("cannot listen on localhost: %v", err)
@ -151,6 +175,10 @@ func NewServer(config *Config) (*Server, error) {
// Quit closes down the server.
func (srv *Server) Quit() {
srv.mu.Lock()
srv.closed = true
srv.mu.Unlock()
srv.listener.Close()
}
@ -175,6 +203,13 @@ func (srv *Server) serveHTTP(w http.ResponseWriter, req *http.Request) {
srv.mu.Lock()
defer srv.mu.Unlock()
if srv.closed {
hj := w.(http.Hijacker)
conn, _, _ := hj.Hijack()
conn.Close()
return
}
if debug {
log.Printf("s3test %q %q", req.Method, req.URL)
}
@ -360,15 +395,28 @@ func (r bucketResource) get(a *action) interface{} {
if maxKeys <= 0 {
maxKeys = 1000
}
resp := &s3.ListResp{
Name: r.bucket.name,
Prefix: prefix,
Delimiter: delimiter,
Marker: marker,
MaxKeys: maxKeys,
type commonPrefix struct {
Prefix string
}
var prefixes []string
type serverListResponse struct {
s3.ListResp
CommonPrefixes []commonPrefix
}
resp := &serverListResponse{
ListResp: s3.ListResp{
Name: r.bucket.name,
Prefix: prefix,
Delimiter: delimiter,
Marker: marker,
MaxKeys: maxKeys,
},
}
var prefixes []commonPrefix
var lastName string
for _, obj := range objs {
if !strings.HasPrefix(obj.name, prefix) {
continue
@ -378,7 +426,7 @@ func (r bucketResource) get(a *action) interface{} {
if delimiter != "" {
if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 {
name = obj.name[:len(prefix)+i+len(delimiter)]
if prefixes != nil && prefixes[len(prefixes)-1] == name {
if prefixes != nil && prefixes[len(prefixes)-1].Prefix == name {
continue
}
isPrefix = true
@ -389,14 +437,16 @@ func (r bucketResource) get(a *action) interface{} {
}
if len(resp.Contents)+len(prefixes) >= maxKeys {
resp.IsTruncated = true
resp.NextMarker = lastName
break
}
if isPrefix {
prefixes = append(prefixes, name)
prefixes = append(prefixes, commonPrefix{Prefix: name})
} else {
// Contents contains only keys not found in CommonPrefixes
resp.Contents = append(resp.Contents, obj.s3Key())
}
lastName = name
}
resp.CommonPrefixes = prefixes
return resp
@ -649,7 +699,7 @@ func (objr objectResource) get(a *action) interface{} {
// TODO x-amz-request-id
h.Set("Content-Length", fmt.Sprint(len(data)))
h.Set("ETag", "\""+hex.EncodeToString(obj.checksum)+"\"")
h.Set("Last-Modified", obj.mtime.Format(lastModifiedTimeFormat))
h.Set("Last-Modified", obj.mtime.UTC().Format(lastModifiedTimeFormat))
if status != http.StatusOK {
a.w.WriteHeader(status)
@ -682,6 +732,8 @@ func (objr objectResource) put(a *action) interface{} {
// TODO x-amz-server-side-encryption
// TODO x-amz-storage-class
var res interface{}
uploadId := a.req.URL.Query().Get("uploadId")
var partNumber uint
@ -751,6 +803,7 @@ func (objr objectResource) put(a *action) interface{} {
obj.meta[key] = values
}
}
obj.mtime = a.srv.config.Clock.Now()
if copySource := a.req.Header.Get("X-Amz-Copy-Source"); copySource != "" {
idx := strings.IndexByte(copySource, '/')
@ -786,11 +839,15 @@ func (objr objectResource) put(a *action) interface{} {
obj.meta[k] = make([]string, len(v))
copy(obj.meta[k], v)
}
res = &s3.CopyObjectResult{
ETag: etag,
LastModified: obj.mtime.UTC().Format(time.RFC3339),
}
} else {
obj.data = data
obj.checksum = gotHash
}
obj.mtime = time.Now()
objr.bucket.objects[objr.name] = obj
} else {
// For multipart commit
@ -800,13 +857,13 @@ func (objr objectResource) put(a *action) interface{} {
index: partNumber,
data: data,
etag: etag,
lastModified: time.Now(),
lastModified: a.srv.config.Clock.Now(),
}
objr.bucket.multipartUploads[uploadId] = append(parts, part)
}
return nil
return res
}
func (objr objectResource) delete(a *action) interface{} {

View file

@ -4,7 +4,7 @@ import (
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"github.com/AdRoll/goamz/aws"
"github.com/docker/goamz/aws"
"log"
"sort"
"strings"

View file

@ -9,6 +9,7 @@ import (
"net/http"
"net/url"
"os"
"runtime"
"time"
log "github.com/Sirupsen/logrus"
@ -31,6 +32,7 @@ import (
storagedriver "github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/factory"
storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware"
"github.com/docker/distribution/version"
"github.com/docker/libtrust"
"github.com/garyburd/redigo/redis"
"github.com/gorilla/mux"
@ -84,12 +86,12 @@ type App struct {
// NewApp takes a configuration and returns a configured app, ready to serve
// requests. The app only implements ServeHTTP and can be wrapped in other
// handlers accordingly.
func NewApp(ctx context.Context, configuration *configuration.Configuration) *App {
func NewApp(ctx context.Context, config *configuration.Configuration) *App {
app := &App{
Config: configuration,
Config: config,
Context: ctx,
router: v2.RouterWithPrefix(configuration.HTTP.Prefix),
isCache: configuration.Proxy.RemoteURL != "",
router: v2.RouterWithPrefix(config.HTTP.Prefix),
isCache: config.Proxy.RemoteURL != "",
}
// Register the handler dispatchers.
@ -103,8 +105,15 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
app.register(v2.RouteNameBlobUpload, blobUploadDispatcher)
app.register(v2.RouteNameBlobUploadChunk, blobUploadDispatcher)
// override the storage driver's UA string for registry outbound HTTP requests
storageParams := config.Storage.Parameters()
if storageParams == nil {
storageParams = make(configuration.Parameters)
}
storageParams["useragent"] = fmt.Sprintf("docker-distribution/%s %s", version.Version, runtime.Version())
var err error
app.driver, err = factory.Create(configuration.Storage.Type(), configuration.Storage.Parameters())
app.driver, err = factory.Create(config.Storage.Type(), storageParams)
if err != nil {
// TODO(stevvooe): Move the creation of a service into a protected
// method, where this is created lazily. Its status can be queried via
@ -113,7 +122,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
}
purgeConfig := uploadPurgeDefaultConfig()
if mc, ok := configuration.Storage["maintenance"]; ok {
if mc, ok := config.Storage["maintenance"]; ok {
if v, ok := mc["uploadpurging"]; ok {
purgeConfig, ok = v.(map[interface{}]interface{})
if !ok {
@ -136,15 +145,15 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
startUploadPurger(app, app.driver, ctxu.GetLogger(app), purgeConfig)
app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"])
app.driver, err = applyStorageMiddleware(app.driver, config.Middleware["storage"])
if err != nil {
panic(err)
}
app.configureSecret(configuration)
app.configureEvents(configuration)
app.configureRedis(configuration)
app.configureLogHook(configuration)
app.configureSecret(config)
app.configureEvents(config)
app.configureRedis(config)
app.configureLogHook(config)
// Generate an ephemeral key to be used for signing converted manifests
// for clients that don't support schema2.
@ -153,8 +162,8 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
panic(err)
}
if configuration.HTTP.Host != "" {
u, err := url.Parse(configuration.HTTP.Host)
if config.HTTP.Host != "" {
u, err := url.Parse(config.HTTP.Host)
if err != nil {
panic(fmt.Sprintf(`could not parse http "host" parameter: %v`, err))
}
@ -168,7 +177,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
}
// configure deletion
if d, ok := configuration.Storage["delete"]; ok {
if d, ok := config.Storage["delete"]; ok {
e, ok := d["enabled"]
if ok {
if deleteEnabled, ok := e.(bool); ok && deleteEnabled {
@ -179,7 +188,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
// configure redirects
var redirectDisabled bool
if redirectConfig, ok := configuration.Storage["redirect"]; ok {
if redirectConfig, ok := config.Storage["redirect"]; ok {
v := redirectConfig["disable"]
switch v := v.(type) {
case bool:
@ -195,7 +204,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
}
// configure storage caches
if cc, ok := configuration.Storage["cache"]; ok {
if cc, ok := config.Storage["cache"]; ok {
v, ok := cc["blobdescriptor"]
if !ok {
// Backwards compatible: "layerinfo" == "blobdescriptor"
@ -224,7 +233,7 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
ctxu.GetLogger(app).Infof("using inmemory blob descriptor cache")
default:
if v != "" {
ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", configuration.Storage["cache"])
ctxu.GetLogger(app).Warnf("unknown cache type %q, caching disabled", config.Storage["cache"])
}
}
}
@ -237,15 +246,15 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
}
}
app.registry, err = applyRegistryMiddleware(app.Context, app.registry, configuration.Middleware["registry"])
app.registry, err = applyRegistryMiddleware(app.Context, app.registry, config.Middleware["registry"])
if err != nil {
panic(err)
}
authType := configuration.Auth.Type()
authType := config.Auth.Type()
if authType != "" {
accessController, err := auth.GetAccessController(configuration.Auth.Type(), configuration.Auth.Parameters())
accessController, err := auth.GetAccessController(config.Auth.Type(), config.Auth.Parameters())
if err != nil {
panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err))
}
@ -254,13 +263,13 @@ func NewApp(ctx context.Context, configuration *configuration.Configuration) *Ap
}
// configure as a pull through cache
if configuration.Proxy.RemoteURL != "" {
app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, configuration.Proxy)
if config.Proxy.RemoteURL != "" {
app.registry, err = proxy.NewRegistryPullThroughCache(ctx, app.registry, app.driver, config.Proxy)
if err != nil {
panic(err.Error())
}
app.isCache = true
ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", configuration.Proxy.RemoteURL)
ctxu.GetLogger(app).Info("Registry configured as a proxy cache to ", config.Proxy.RemoteURL)
}
return app

View file

@ -10,10 +10,10 @@ import (
"io/ioutil"
"time"
"github.com/AdRoll/goamz/cloudfront"
"github.com/docker/distribution/context"
storagedriver "github.com/docker/distribution/registry/storage/driver"
storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware"
"github.com/docker/goamz/cloudfront"
)
// cloudFrontStorageMiddleware provides an simple implementation of layerHandler that

View file

@ -26,11 +26,12 @@ import (
"sync"
"time"
"github.com/AdRoll/goamz/aws"
"github.com/AdRoll/goamz/s3"
"github.com/Sirupsen/logrus"
"github.com/docker/goamz/aws"
"github.com/docker/goamz/s3"
"github.com/docker/distribution/context"
"github.com/docker/distribution/registry/client/transport"
storagedriver "github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/base"
"github.com/docker/distribution/registry/storage/driver/factory"
@ -58,6 +59,7 @@ type DriverParameters struct {
V4Auth bool
ChunkSize int64
RootDirectory string
UserAgent string
}
func init() {
@ -168,7 +170,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
case int, uint, int32, uint32, uint64:
chunkSize = reflect.ValueOf(v).Convert(reflect.TypeOf(chunkSize)).Int()
default:
return nil, fmt.Errorf("invalid valud for chunksize: %#v", chunkSizeParam)
return nil, fmt.Errorf("invalid value for chunksize: %#v", chunkSizeParam)
}
if chunkSize < minChunkSize {
@ -181,6 +183,11 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
rootDirectory = ""
}
userAgent, ok := parameters["useragent"]
if !ok {
userAgent = ""
}
params := DriverParameters{
fmt.Sprint(accessKey),
fmt.Sprint(secretKey),
@ -191,6 +198,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
v4AuthBool,
chunkSize,
fmt.Sprint(rootDirectory),
fmt.Sprint(userAgent),
}
return New(params)
@ -209,7 +217,16 @@ func New(params DriverParameters) (*Driver, error) {
}
s3obj := s3.New(auth, params.Region)
bucket := s3obj.Bucket(params.Bucket)
if params.UserAgent != "" {
s3obj.Client = &http.Client{
Transport: transport.NewTransport(http.DefaultTransport,
transport.NewHeaderRequestModifier(http.Header{
http.CanonicalHeaderKey("User-Agent"): []string{params.UserAgent},
}),
),
}
}
if params.V4Auth {
s3obj.Signature = aws.V4Signature
@ -219,6 +236,8 @@ func New(params DriverParameters) (*Driver, error) {
}
}
bucket := s3obj.Bucket(params.Bucket)
// TODO Currently multipart uploads have no timestamps, so this would be unwise
// if you initiated a new s3driver while another one is running on the same bucket.
// multis, _, err := bucket.ListMulti("", "")

View file

@ -6,10 +6,10 @@ import (
"strconv"
"testing"
"github.com/AdRoll/goamz/aws"
"github.com/docker/distribution/context"
storagedriver "github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/testsuites"
"github.com/docker/goamz/aws"
"gopkg.in/check.v1"
)
@ -69,6 +69,7 @@ func init() {
v4AuthBool,
minChunkSize,
rootDirectory,
"",
}
return New(parameters)