Vendor update for aws sdk

Updated to latest version of go aws sdk.
Use vendored sub pakages within aws sdk.
Adds missing vendor packages for letsencrypt

Fixes #1832

Signed-off-by: Derek McGowan <derek@mcgstyle.net> (github: dmcgowan)
This commit is contained in:
Derek McGowan 2016-07-12 12:47:37 -07:00
parent 2052f29be6
commit acae5dcfff
64 changed files with 5659 additions and 1382 deletions

View file

@ -13,6 +13,7 @@ import (
"net/url"
"strings"
"time"
"unicode"
)
// An AWSEpochTime wraps a time value providing JSON serialization needed for
@ -110,6 +111,12 @@ func (p *Policy) Validate() error {
if s.Resource == "" {
return fmt.Errorf("statement at index %d does not have a resource", i)
}
if !isASCII(s.Resource) {
return fmt.Errorf("unable to sign resource, [%s]. "+
"Resources must only contain ascii characters. "+
"Hostnames with unicode should be encoded as Punycode, (e.g. golang.org/x/net/idna), "+
"and URL unicode path/query characters should be escaped.", s.Resource)
}
}
return nil
@ -120,7 +127,7 @@ func (p *Policy) Validate() error {
func CreateResource(scheme, u string) (string, error) {
scheme = strings.ToLower(scheme)
if scheme == "http" || scheme == "https" {
if scheme == "http" || scheme == "https" || scheme == "http*" || scheme == "*" {
return u, nil
}
@ -208,3 +215,12 @@ func awsEscapeEncoded(b []byte) {
}
}
}
func isASCII(u string) bool {
for _, c := range u {
if c > unicode.MaxASCII {
return false
}
}
return true
}

View file

@ -0,0 +1,241 @@
package sign
import (
"crypto/rsa"
"fmt"
"net/http"
"strings"
"time"
)
const (
// CookiePolicyName name of the policy cookie
CookiePolicyName = "CloudFront-Policy"
// CookieSignatureName name of the signature cookie
CookieSignatureName = "CloudFront-Signature"
// CookieKeyIDName name of the signing Key ID cookie
CookieKeyIDName = "CloudFront-Key-Pair-Id"
)
// A CookieOptions optional additonal options that can be applied to the signed
// cookies.
type CookieOptions struct {
Path string
Domain string
Secure bool
}
// apply will integration the options provided into the base cookie options
// a new copy will be returned. The base CookieOption will not be modified.
func (o CookieOptions) apply(opts ...func(*CookieOptions)) CookieOptions {
if len(opts) == 0 {
return o
}
for _, opt := range opts {
opt(&o)
}
return o
}
// A CookieSigner provides signing utilities to sign Cookies for Amazon CloudFront
// resources. Using a private key and Credential Key Pair key ID the CookieSigner
// only needs to be created once per Credential Key Pair key ID and private key.
//
// More information about signed Cookies and their structure can be found at:
// http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-setting-signed-cookie-custom-policy.html
//
// To sign a Cookie, create a CookieSigner with your private key and credential
// pair key ID. Once you have a CookieSigner instance you can call Sign or
// SignWithPolicy to sign the URLs.
//
// The signer is safe to use concurrently, but the optional cookies options
// are not safe to modify concurrently.
type CookieSigner struct {
keyID string
privKey *rsa.PrivateKey
Opts CookieOptions
}
// NewCookieSigner constructs and returns a new CookieSigner to be used to for
// signing Amazon CloudFront URL resources with.
func NewCookieSigner(keyID string, privKey *rsa.PrivateKey, opts ...func(*CookieOptions)) *CookieSigner {
signer := &CookieSigner{
keyID: keyID,
privKey: privKey,
Opts: CookieOptions{}.apply(opts...),
}
return signer
}
// Sign returns the cookies needed to allow user agents to make arbetrary
// requests to cloudfront for the resource(s) defined by the policy.
//
// Sign will create a CloudFront policy with only a resource and condition of
// DateLessThan equal to the expires time provided.
//
// The returned slice cookies should all be added to the Client's cookies or
// server's response.
//
// Example:
// s := NewCookieSigner(keyID, privKey)
//
// // Get Signed cookies for a resource that will expire in 1 hour
// cookies, err := s.Sign("*", time.Now().Add(1 * time.Hour))
// if err != nil {
// fmt.Println("failed to create signed cookies", err)
// return
// }
//
// // Or get Signed cookies for a resource that will expire in 1 hour
// // and set path and domain of cookies
// cookies, err := s.Sign("*", time.Now().Add(1 * time.Hour), func(o *sign.CookieOptions) {
// o.Path = "/"
// o.Domain = ".example.com"
// })
// if err != nil {
// fmt.Println("failed to create signed cookies", err)
// return
// }
//
// // Server Response via http.ResponseWriter
// for _, c := range cookies {
// http.SetCookie(w, c)
// }
//
// // Client request via the cookie jar
// if client.CookieJar != nil {
// for _, c := range cookies {
// client.Cookie(w, c)
// }
// }
func (s CookieSigner) Sign(u string, expires time.Time, opts ...func(*CookieOptions)) ([]*http.Cookie, error) {
scheme, err := cookieURLScheme(u)
if err != nil {
return nil, err
}
resource, err := CreateResource(scheme, u)
if err != nil {
return nil, err
}
p := NewCannedPolicy(resource, expires)
return createCookies(p, s.keyID, s.privKey, s.Opts.apply(opts...))
}
// Returns and validates the URL's scheme.
// http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-setting-signed-cookie-custom-policy.html#private-content-custom-policy-statement-cookies
func cookieURLScheme(u string) (string, error) {
parts := strings.SplitN(u, "://", 2)
if len(parts) != 2 {
return "", fmt.Errorf("invalid cookie URL, missing scheme")
}
scheme := strings.ToLower(parts[0])
if scheme != "http" && scheme != "https" && scheme != "http*" {
return "", fmt.Errorf("invalid cookie URL scheme. Expect http, https, or http*. Go, %s", scheme)
}
return scheme, nil
}
// SignWithPolicy returns the cookies needed to allow user agents to make
// arbetrairy requets to cloudfront for the resource(s) defined by the policy.
//
// The returned slice cookies should all be added to the Client's cookies or
// server's response.
//
// Example:
// s := NewCookieSigner(keyID, privKey)
//
// policy := &sign.Policy{
// Statements: []sign.Statement{
// {
// // Read the provided documentation on how to set this
// // correctly, you'll probably want to use wildcards.
// Resource: RawCloudFrontURL,
// Condition: sign.Condition{
// // Optional IP source address range
// IPAddress: &sign.IPAddress{SourceIP: "192.0.2.0/24"},
// // Optional date URL is not valid until
// DateGreaterThan: &sign.AWSEpochTime{time.Now().Add(30 * time.Minute)},
// // Required date the URL will expire after
// DateLessThan: &sign.AWSEpochTime{time.Now().Add(1 * time.Hour)},
// },
// },
// },
// }
//
// // Get Signed cookies for a resource that will expire in 1 hour
// cookies, err := s.SignWithPolicy(policy)
// if err != nil {
// fmt.Println("failed to create signed cookies", err)
// return
// }
//
// // Or get Signed cookies for a resource that will expire in 1 hour
// // and set path and domain of cookies
// cookies, err := s.Sign(policy, func(o *sign.CookieOptions) {
// o.Path = "/"
// o.Domain = ".example.com"
// })
// if err != nil {
// fmt.Println("failed to create signed cookies", err)
// return
// }
//
// // Server Response via http.ResponseWriter
// for _, c := range cookies {
// http.SetCookie(w, c)
// }
//
// // Client request via the cookie jar
// if client.CookieJar != nil {
// for _, c := range cookies {
// client.Cookie(w, c)
// }
// }
func (s CookieSigner) SignWithPolicy(p *Policy, opts ...func(*CookieOptions)) ([]*http.Cookie, error) {
return createCookies(p, s.keyID, s.privKey, s.Opts.apply(opts...))
}
// Prepares the cookies to be attached to the header. An (optional) options
// struct is provided in case people don't want to manually edit their cookies.
func createCookies(p *Policy, keyID string, privKey *rsa.PrivateKey, opt CookieOptions) ([]*http.Cookie, error) {
b64Sig, b64Policy, err := p.Sign(privKey)
if err != nil {
return nil, err
}
// Creates proper cookies
cPolicy := &http.Cookie{
Name: CookiePolicyName,
Value: string(b64Policy),
HttpOnly: true,
}
cSignature := &http.Cookie{
Name: CookieSignatureName,
Value: string(b64Sig),
HttpOnly: true,
}
cKey := &http.Cookie{
Name: CookieKeyIDName,
Value: keyID,
HttpOnly: true,
}
cookies := []*http.Cookie{cPolicy, cSignature, cKey}
// Applie the cookie options
for _, c := range cookies {
c.Path = opt.Path
c.Domain = opt.Domain
c.Secure = opt.Secure
}
return cookies, nil
}

View file

@ -90,19 +90,19 @@ func (s URLSigner) Sign(url string, expires time.Time) (string, error) {
// // Sign URL to be valid for 30 minutes from now, expires one hour from now, and
// // restricted to the 192.0.2.0/24 IP address range.
// policy := &sign.Policy{
// Statements: []Statement{
// Statements: []sign.Statement{
// {
// Resource: rawURL,
// Condition: Condition{
// Condition: sign.Condition{
// // Optional IP source address range
// IPAddress: &IPAddress{SourceIP: "192.0.2.0/24"},
// IPAddress: &sign.IPAddress{SourceIP: "192.0.2.0/24"},
// // Optional date URL is not valid until
// DateGreaterThan: &AWSEpochTime{time.Now().Add(30 * time.Minute)},
// DateGreaterThan: &sign.AWSEpochTime{time.Now().Add(30 * time.Minute)},
// // Required date the URL will expire after
// DateLessThan: &AWSEpochTime{time.Now().Add(1 * time.Hour)},
// }
// }
// }
// DateLessThan: &sign.AWSEpochTime{time.Now().Add(1 * time.Hour)},
// },
// },
// },
// }
//
// signer := sign.NewURLSigner(keyID, privKey)

File diff suppressed because it is too large Load diff

View file

@ -6,32 +6,41 @@ import (
)
func init() {
initClient = func(c *client.Client) {
// Support building custom host-style bucket endpoints
c.Handlers.Build.PushFront(updateHostWithBucket)
initClient = defaultInitClientFn
initRequest = defaultInitRequestFn
}
// Require SSL when using SSE keys
c.Handlers.Validate.PushBack(validateSSERequiresSSL)
c.Handlers.Build.PushBack(computeSSEKeys)
func defaultInitClientFn(c *client.Client) {
// Support building custom endpoints based on config
c.Handlers.Build.PushFront(updateEndpointForS3Config)
// S3 uses custom error unmarshaling logic
c.Handlers.UnmarshalError.Clear()
c.Handlers.UnmarshalError.PushBack(unmarshalError)
}
// Require SSL when using SSE keys
c.Handlers.Validate.PushBack(validateSSERequiresSSL)
c.Handlers.Build.PushBack(computeSSEKeys)
initRequest = func(r *request.Request) {
switch r.Operation.Name {
case opPutBucketCors, opPutBucketLifecycle, opPutBucketPolicy, opPutBucketTagging, opDeleteObjects, opPutBucketLifecycleConfiguration:
// These S3 operations require Content-MD5 to be set
r.Handlers.Build.PushBack(contentMD5)
case opGetBucketLocation:
// GetBucketLocation has custom parsing logic
r.Handlers.Unmarshal.PushFront(buildGetBucketLocation)
case opCreateBucket:
// Auto-populate LocationConstraint with current region
r.Handlers.Validate.PushFront(populateLocationConstraint)
case opCopyObject, opUploadPartCopy, opCompleteMultipartUpload:
r.Handlers.Unmarshal.PushFront(copyMultipartStatusOKUnmarhsalError)
}
// S3 uses custom error unmarshaling logic
c.Handlers.UnmarshalError.Clear()
c.Handlers.UnmarshalError.PushBack(unmarshalError)
}
func defaultInitRequestFn(r *request.Request) {
// Add reuest handlers for specific platforms.
// e.g. 100-continue support for PUT requests using Go 1.6
platformRequestHandlers(r)
switch r.Operation.Name {
case opPutBucketCors, opPutBucketLifecycle, opPutBucketPolicy,
opPutBucketTagging, opDeleteObjects, opPutBucketLifecycleConfiguration,
opPutBucketReplication:
// These S3 operations require Content-MD5 to be set
r.Handlers.Build.PushBack(contentMD5)
case opGetBucketLocation:
// GetBucketLocation has custom parsing logic
r.Handlers.Unmarshal.PushFront(buildGetBucketLocation)
case opCreateBucket:
// Auto-populate LocationConstraint with current region
r.Handlers.Validate.PushFront(populateLocationConstraint)
case opCopyObject, opUploadPartCopy, opCompleteMultipartUpload:
r.Handlers.Unmarshal.PushFront(copyMultipartStatusOKUnmarhsalError)
}
}

View file

@ -1,14 +1,124 @@
package s3
import (
"fmt"
"net/url"
"regexp"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/aws/request"
)
// an operationBlacklist is a list of operation names that should a
// request handler should not be executed with.
type operationBlacklist []string
// Continue will return true of the Request's operation name is not
// in the blacklist. False otherwise.
func (b operationBlacklist) Continue(r *request.Request) bool {
for i := 0; i < len(b); i++ {
if b[i] == r.Operation.Name {
return false
}
}
return true
}
var accelerateOpBlacklist = operationBlacklist{
opListBuckets, opCreateBucket, opDeleteBucket,
}
// Request handler to automatically add the bucket name to the endpoint domain
// if possible. This style of bucket is valid for all bucket names which are
// DNS compatible and do not contain "."
func updateEndpointForS3Config(r *request.Request) {
forceHostStyle := aws.BoolValue(r.Config.S3ForcePathStyle)
accelerate := aws.BoolValue(r.Config.S3UseAccelerate)
if accelerate && accelerateOpBlacklist.Continue(r) {
if forceHostStyle {
if r.Config.Logger != nil {
r.Config.Logger.Log("ERROR: aws.Config.S3UseAccelerate is not compatible with aws.Config.S3ForcePathStyle, ignoring S3ForcePathStyle.")
}
}
updateEndpointForAccelerate(r)
} else if !forceHostStyle && r.Operation.Name != opGetBucketLocation {
updateEndpointForHostStyle(r)
}
}
func updateEndpointForHostStyle(r *request.Request) {
bucket, ok := bucketNameFromReqParams(r.Params)
if !ok {
// Ignore operation requests if the bucketname was not provided
// if this is an input validation error the validation handler
// will report it.
return
}
if !hostCompatibleBucketName(r.HTTPRequest.URL, bucket) {
// bucket name must be valid to put into the host
return
}
moveBucketToHost(r.HTTPRequest.URL, bucket)
}
func updateEndpointForAccelerate(r *request.Request) {
bucket, ok := bucketNameFromReqParams(r.Params)
if !ok {
// Ignore operation requests if the bucketname was not provided
// if this is an input validation error the validation handler
// will report it.
return
}
if !hostCompatibleBucketName(r.HTTPRequest.URL, bucket) {
r.Error = awserr.New("InvalidParameterException",
fmt.Sprintf("bucket name %s is not compatibile with S3 Accelerate", bucket),
nil)
return
}
// Change endpoint from s3(-[a-z0-1-])?.amazonaws.com to s3-accelerate.amazonaws.com
r.HTTPRequest.URL.Host = replaceHostRegion(r.HTTPRequest.URL.Host, "accelerate")
moveBucketToHost(r.HTTPRequest.URL, bucket)
}
// Attempts to retrieve the bucket name from the request input parameters.
// If no bucket is found, or the field is empty "", false will be returned.
func bucketNameFromReqParams(params interface{}) (string, bool) {
b, _ := awsutil.ValuesAtPath(params, "Bucket")
if len(b) == 0 {
return "", false
}
if bucket, ok := b[0].(*string); ok {
if bucketStr := aws.StringValue(bucket); bucketStr != "" {
return bucketStr, true
}
}
return "", false
}
// hostCompatibleBucketName returns true if the request should
// put the bucket in the host. This is false if S3ForcePathStyle is
// explicitly set or if the bucket is not DNS compatible.
func hostCompatibleBucketName(u *url.URL, bucket string) bool {
// Bucket might be DNS compatible but dots in the hostname will fail
// certificate validation, so do not use host-style.
if u.Scheme == "https" && strings.Contains(bucket, ".") {
return false
}
// if the bucket is DNS compatible
return dnsCompatibleBucketName(bucket)
}
var reDomain = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
var reIPAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`)
@ -20,41 +130,36 @@ func dnsCompatibleBucketName(bucket string) bool {
!strings.Contains(bucket, "..")
}
// hostStyleBucketName returns true if the request should put the bucket in
// the host. This is false if S3ForcePathStyle is explicitly set or if the
// bucket is not DNS compatible.
func hostStyleBucketName(r *request.Request, bucket string) bool {
if aws.BoolValue(r.Config.S3ForcePathStyle) {
return false
// moveBucketToHost moves the bucket name from the URI path to URL host.
func moveBucketToHost(u *url.URL, bucket string) {
u.Host = bucket + "." + u.Host
u.Path = strings.Replace(u.Path, "/{Bucket}", "", -1)
if u.Path == "" {
u.Path = "/"
}
// Bucket might be DNS compatible but dots in the hostname will fail
// certificate validation, so do not use host-style.
if r.HTTPRequest.URL.Scheme == "https" && strings.Contains(bucket, ".") {
return false
}
// GetBucketLocation should be able to be called from any region within
// a partition, and return the associated region of the bucket.
if r.Operation.Name == opGetBucketLocation {
return false
}
// Use host-style if the bucket is DNS compatible
return dnsCompatibleBucketName(bucket)
}
func updateHostWithBucket(r *request.Request) {
b, _ := awsutil.ValuesAtPath(r.Params, "Bucket")
if len(b) == 0 {
return
const s3HostPrefix = "s3"
// replaceHostRegion replaces the S3 region string in the host with the
// value provided. If v is empty the host prefix returned will be s3.
func replaceHostRegion(host, v string) string {
if !strings.HasPrefix(host, s3HostPrefix) {
return host
}
if bucket := b[0].(*string); aws.StringValue(bucket) != "" && hostStyleBucketName(r, *bucket) {
r.HTTPRequest.URL.Host = *bucket + "." + r.HTTPRequest.URL.Host
r.HTTPRequest.URL.Path = strings.Replace(r.HTTPRequest.URL.Path, "/{Bucket}", "", -1)
if r.HTTPRequest.URL.Path == "" {
r.HTTPRequest.URL.Path = "/"
suffix := host[len(s3HostPrefix):]
for i := len(s3HostPrefix); i < len(host); i++ {
if host[i] == '.' {
// Trim until '.' leave the it in place.
suffix = host[i:]
break
}
}
if len(v) == 0 {
return fmt.Sprintf("s3%s", suffix)
}
return fmt.Sprintf("s3-%s%s", v, suffix)
}

View file

@ -0,0 +1,8 @@
// +build !go1.6
package s3
import "github.com/aws/aws-sdk-go/aws/request"
func platformRequestHandlers(r *request.Request) {
}

View file

@ -0,0 +1,28 @@
// +build go1.6
package s3
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
)
func platformRequestHandlers(r *request.Request) {
if r.Operation.HTTPMethod == "PUT" {
// 100-Continue should only be used on put requests.
r.Handlers.Sign.PushBack(add100Continue)
}
}
func add100Continue(r *request.Request) {
if aws.BoolValue(r.Config.S3Disable100Continue) {
return
}
if r.HTTPRequest.ContentLength < 1024*1024*2 {
// Ignore requests smaller than 2MB. This helps prevent delaying
// requests unnecessarily.
return
}
r.HTTPRequest.Header.Set("Expect", "100-Continue")
}

View file

@ -7,8 +7,8 @@ import (
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/client/metadata"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/aws/aws-sdk-go/private/protocol/restxml"
"github.com/aws/aws-sdk-go/private/signer/v4"
)
// S3 is a client for Amazon S3.
@ -58,7 +58,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
}
// Handlers
svc.Handlers.Sign.PushBack(v4.Sign)
svc.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
svc.Handlers.Build.PushBackNamed(restxml.BuildHandler)
svc.Handlers.Unmarshal.PushBackNamed(restxml.UnmarshalHandler)
svc.Handlers.UnmarshalMeta.PushBackNamed(restxml.UnmarshalMetaHandler)

View file

@ -4,6 +4,7 @@ import (
"encoding/xml"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
@ -20,6 +21,7 @@ type xmlErrorResponse struct {
func unmarshalError(r *request.Request) {
defer r.HTTPResponse.Body.Close()
defer io.Copy(ioutil.Discard, r.HTTPResponse.Body)
if r.HTTPResponse.StatusCode == http.StatusMovedPermanently {
r.Error = awserr.NewRequestFailure(

View file

@ -18,6 +18,12 @@ func (c *S3) WaitUntilBucketExists(input *HeadBucketInput) error {
Argument: "",
Expected: 200,
},
{
State: "success",
Matcher: "status",
Argument: "",
Expected: 301,
},
{
State: "success",
Matcher: "status",