From 5ad9d19ff603945b239cc466869630cc06ce46d5 Mon Sep 17 00:00:00 2001 From: Cezar Sa Espinola Date: Mon, 23 May 2016 16:31:57 -0300 Subject: [PATCH 1/3] Update vendored version of ncw/swift library Signed-off-by: Cezar Sa Espinola --- Godeps/Godeps.json | 6 +- vendor/github.com/ncw/swift/.travis.yml | 5 +- vendor/github.com/ncw/swift/README.md | 5 +- vendor/github.com/ncw/swift/auth.go | 46 ++++++++++++-- vendor/github.com/ncw/swift/auth_v3.go | 47 ++++++++------ vendor/github.com/ncw/swift/swift.go | 83 ++++++++++++++----------- 6 files changed, 125 insertions(+), 67 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 8597ea43..9e739e0d 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "github.com/docker/distribution", "GoVersion": "go1.6", - "GodepVersion": "v60", + "GodepVersion": "v70", "Packages": [ "./..." ], @@ -223,11 +223,11 @@ }, { "ImportPath": "github.com/ncw/swift", - "Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3" + "Rev": "ce444d6d47c51d4dda9202cd38f5094dd8e27e86" }, { "ImportPath": "github.com/ncw/swift/swifttest", - "Rev": "c54732e87b0b283d1baf0a18db689d0aea460ba3" + "Rev": "ce444d6d47c51d4dda9202cd38f5094dd8e27e86" }, { "ImportPath": "github.com/spf13/cobra", diff --git a/vendor/github.com/ncw/swift/.travis.yml b/vendor/github.com/ncw/swift/.travis.yml index 4e7c3b3b..bf37e41f 100644 --- a/vendor/github.com/ncw/swift/.travis.yml +++ b/vendor/github.com/ncw/swift/.travis.yml @@ -4,8 +4,11 @@ sudo: false go: - 1.1.2 - 1.2.2 - - 1.3 + - 1.3.3 + - 1.4.2 + - 1.5.1 - tip script: + - test -z "$(go fmt ./...)" - go test diff --git a/vendor/github.com/ncw/swift/README.md b/vendor/github.com/ncw/swift/README.md index f51266a2..3855a394 100644 --- a/vendor/github.com/ncw/swift/README.md +++ b/vendor/github.com/ncw/swift/README.md @@ -9,7 +9,7 @@ See here for package docs http://godoc.org/github.com/ncw/swift -[![Build Status](https://travis-ci.org/ncw/swift.png)](https://travis-ci.org/ncw/swift) +[![Build Status](https://api.travis-ci.org/ncw/swift.svg?branch=master)](https://travis-ci.org/ncw/swift) [![GoDoc](https://godoc.org/github.com/ncw/swift?status.svg)](https://godoc.org/github.com/ncw/swift) Install ------- @@ -134,3 +134,6 @@ Contributors - Fabian Ruff - Arturo Reuschenbach Puncernau - Petr Kotek +- Stefan Majewsky +- Cezar Sa Espinola +- Sam Gunaratne diff --git a/vendor/github.com/ncw/swift/auth.go b/vendor/github.com/ncw/swift/auth.go index 3948bc1a..316dc7fe 100644 --- a/vendor/github.com/ncw/swift/auth.go +++ b/vendor/github.com/ncw/swift/auth.go @@ -12,7 +12,9 @@ import ( // // This encapsulates the different authentication schemes in use type Authenticator interface { + // Request creates an http.Request for the auth - return nil if not needed Request(*Connection) (*http.Request, error) + // Response parses the http.Response Response(resp *http.Response) error // The public storage URL - set Internal to true to read // internal/service net URL @@ -23,6 +25,23 @@ type Authenticator interface { CdnUrl() string } +type CustomEndpointAuthenticator interface { + StorageUrlForEndpoint(endpointType EndpointType) string +} + +type EndpointType string + +const ( + // Use public URL as storage URL + EndpointTypePublic = EndpointType("public") + + // Use internal URL as storage URL + EndpointTypeInternal = EndpointType("internal") + + // Use admin URL as storage URL + EndpointTypeAdmin = EndpointType("admin") +) + // newAuth - create a new Authenticator from the AuthUrl // // A hint for AuthVersion can be provided @@ -175,15 +194,20 @@ func (auth *v2Auth) Response(resp *http.Response) error { // Region if set or defaulting to the first one if not // // Returns "" if not found -func (auth *v2Auth) endpointUrl(Type string, Internal bool) string { +func (auth *v2Auth) endpointUrl(Type string, endpointType EndpointType) string { for _, catalog := range auth.Auth.Access.ServiceCatalog { if catalog.Type == Type { for _, endpoint := range catalog.Endpoints { if auth.Region == "" || (auth.Region == endpoint.Region) { - if Internal { + switch endpointType { + case EndpointTypeInternal: return endpoint.InternalUrl - } else { + case EndpointTypePublic: return endpoint.PublicUrl + case EndpointTypeAdmin: + return endpoint.AdminUrl + default: + return "" } } } @@ -197,7 +221,18 @@ func (auth *v2Auth) endpointUrl(Type string, Internal bool) string { // If Internal is true then it reads the private (internal / service // net) URL. func (auth *v2Auth) StorageUrl(Internal bool) string { - return auth.endpointUrl("object-store", Internal) + endpointType := EndpointTypePublic + if Internal { + endpointType = EndpointTypeInternal + } + return auth.StorageUrlForEndpoint(endpointType) +} + +// v2 Authentication - read storage url +// +// Use the indicated endpointType to choose a URL. +func (auth *v2Auth) StorageUrlForEndpoint(endpointType EndpointType) string { + return auth.endpointUrl("object-store", endpointType) } // v2 Authentication - read auth token @@ -207,7 +242,7 @@ func (auth *v2Auth) Token() string { // v2 Authentication - read cdn url func (auth *v2Auth) CdnUrl() string { - return auth.endpointUrl("rax:object-cdn", false) + return auth.endpointUrl("rax:object-cdn", EndpointTypePublic) } // ------------------------------------------------------------ @@ -255,6 +290,7 @@ type v2AuthResponse struct { Endpoints []struct { InternalUrl string PublicUrl string + AdminUrl string Region string TenantId string } diff --git a/vendor/github.com/ncw/swift/auth_v3.go b/vendor/github.com/ncw/swift/auth_v3.go index b657747a..21e96718 100644 --- a/vendor/github.com/ncw/swift/auth_v3.go +++ b/vendor/github.com/ncw/swift/auth_v3.go @@ -10,9 +10,6 @@ import ( const ( v3AuthMethodToken = "token" v3AuthMethodPassword = "password" - v3InterfacePublic = "public" - v3InterfaceInternal = "internal" - v3InterfaceAdmin = "admin" v3CatalogTypeObjectStore = "object-store" ) @@ -88,7 +85,8 @@ type v3AuthResponse struct { Catalog []struct { Id, Namem, Type string Endpoints []struct { - Id, Region_Id, Url, Region, Interface string + Id, Region_Id, Url, Region string + Interface EndpointType } } @@ -107,11 +105,13 @@ type v3AuthResponse struct { } type v3Auth struct { + Region string Auth *v3AuthResponse Headers http.Header } func (auth *v3Auth) Request(c *Connection) (*http.Request, error) { + auth.Region = c.Region var v3i interface{} @@ -149,13 +149,18 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) { v3.Auth.Scope.Project.Id = c.TenantId } else if c.Tenant != "" { v3.Auth.Scope.Project.Name = c.Tenant - var defaultDomain v3Domain - if c.Domain != "" { - defaultDomain = v3Domain{Name: "Default"} - } else if c.DomainId != "" { - defaultDomain = v3Domain{Id: "Default"} + switch { + case c.TenantDomain != "": + v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.TenantDomain} + case c.TenantDomainId != "": + v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.TenantDomainId} + case c.Domain != "": + v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.Domain} + case c.DomainId != "": + v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.DomainId} + default: + v3.Auth.Scope.Project.Domain = &v3Domain{Name: "Default"} } - v3.Auth.Scope.Project.Domain = &defaultDomain } } @@ -188,18 +193,12 @@ func (auth *v3Auth) Response(resp *http.Response) error { return err } -func (auth *v3Auth) endpointUrl(Type string, Internal bool) string { +func (auth *v3Auth) endpointUrl(Type string, endpointType EndpointType) string { for _, catalog := range auth.Auth.Token.Catalog { if catalog.Type == Type { for _, endpoint := range catalog.Endpoints { - if Internal { - if endpoint.Interface == v3InterfaceInternal { - return endpoint.Url - } - } else { - if endpoint.Interface == v3InterfacePublic { - return endpoint.Url - } + if endpoint.Interface == endpointType && (auth.Region == "" || (auth.Region == endpoint.Region)) { + return endpoint.Url } } } @@ -208,7 +207,15 @@ func (auth *v3Auth) endpointUrl(Type string, Internal bool) string { } func (auth *v3Auth) StorageUrl(Internal bool) string { - return auth.endpointUrl(v3CatalogTypeObjectStore, Internal) + endpointType := EndpointTypePublic + if Internal { + endpointType = EndpointTypeInternal + } + return auth.StorageUrlForEndpoint(endpointType) +} + +func (auth *v3Auth) StorageUrlForEndpoint(endpointType EndpointType) string { + return auth.endpointUrl("object-store", endpointType) } func (auth *v3Auth) Token() string { diff --git a/vendor/github.com/ncw/swift/swift.go b/vendor/github.com/ncw/swift/swift.go index 1ad8fcc1..1337f10d 100644 --- a/vendor/github.com/ncw/swift/swift.go +++ b/vendor/github.com/ncw/swift/swift.go @@ -92,11 +92,14 @@ type Connection struct { UserAgent string // Http User agent (default goswift/1.0) ConnectTimeout time.Duration // Connect channel timeout (default 10s) Timeout time.Duration // Data channel timeout (default 60s) - Region string // Region to use eg "LON", "ORD" - default is use first region (V2 auth only) - AuthVersion int // Set to 1 or 2 or leave at 0 for autodetect + Region string // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only) + AuthVersion int // Set to 1, 2 or 3 or leave at 0 for autodetect Internal bool // Set this to true to use the the internal / service network - Tenant string // Name of the tenant (v2 auth only) - TenantId string // Id of the tenant (v2 auth only) + Tenant string // Name of the tenant (v2,v3 auth only) + TenantId string // Id of the tenant (v2,v3 auth only) + EndpointType EndpointType // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set) + TenantDomain string // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain + TenantDomainId string // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain TrustId string // Id of the trust (v3 auth only) Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine) // These are filled in after Authenticate is called as are the defaults for above @@ -300,33 +303,39 @@ again: if err != nil { return } - timer := time.NewTimer(c.ConnectTimeout) - var resp *http.Response - resp, err = c.doTimeoutRequest(timer, req) - if err != nil { - return - } - defer func() { - checkClose(resp.Body, &err) - // Flush the auth connection - we don't want to keep - // it open if keepalives were enabled - flushKeepaliveConnections(c.Transport) - }() - if err = c.parseHeaders(resp, authErrorMap); err != nil { - // Try again for a limited number of times on - // AuthorizationFailed or BadRequest. This allows us - // to try some alternate forms of the request - if (err == AuthorizationFailed || err == BadRequest) && retries > 0 { - retries-- - goto again + if req != nil { + timer := time.NewTimer(c.ConnectTimeout) + var resp *http.Response + resp, err = c.doTimeoutRequest(timer, req) + if err != nil { + return + } + defer func() { + checkClose(resp.Body, &err) + // Flush the auth connection - we don't want to keep + // it open if keepalives were enabled + flushKeepaliveConnections(c.Transport) + }() + if err = c.parseHeaders(resp, authErrorMap); err != nil { + // Try again for a limited number of times on + // AuthorizationFailed or BadRequest. This allows us + // to try some alternate forms of the request + if (err == AuthorizationFailed || err == BadRequest) && retries > 0 { + retries-- + goto again + } + return + } + err = c.Auth.Response(resp) + if err != nil { + return } - return } - err = c.Auth.Response(resp) - if err != nil { - return + if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" { + c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType) + } else { + c.StorageUrl = c.Auth.StorageUrl(c.Internal) } - c.StorageUrl = c.Auth.StorageUrl(c.Internal) c.AuthToken = c.Auth.Token() if !c.authenticated() { err = newError(0, "Response didn't have storage url and auth token") @@ -1424,13 +1433,13 @@ var _ io.Seeker = &ObjectOpenFile{} // will also check the length returned. No checking will be done if // you don't read all the contents. // -// Note that objects with X-Object-Manifest set won't ever have their -// md5sum's checked as the md5sum reported on the object is actually -// the md5sum of the md5sums of the parts. This isn't very helpful to -// detect a corrupted download as the size of the parts aren't known -// without doing more operations. If you want to ensure integrity of -// an object with a manifest then you will need to download everything -// in the manifest separately. +// Note that objects with X-Object-Manifest or X-Static-Large-Object +// set won't ever have their md5sum's checked as the md5sum reported +// on the object is actually the md5sum of the md5sums of the +// parts. This isn't very helpful to detect a corrupted download as +// the size of the parts aren't known without doing more operations. +// If you want to ensure integrity of an object with a manifest then +// you will need to download everything in the manifest separately. // // headers["Content-Type"] will give the content type if desired. func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) { @@ -1445,8 +1454,8 @@ func (c *Connection) ObjectOpen(container string, objectName string, checkHash b if err != nil { return } - // Can't check MD5 on an object with X-Object-Manifest set - if checkHash && headers["X-Object-Manifest"] != "" { + // Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set + if checkHash && (headers["X-Object-Manifest"] != "" || headers["X-Static-Large-Object"] != "") { // log.Printf("swift: turning off md5 checking on object with manifest %v", objectName) checkHash = false } From 7f72092940c3559eb662ef1e3c369c3795e7b254 Mon Sep 17 00:00:00 2001 From: Cezar Sa Espinola Date: Mon, 7 Mar 2016 18:41:20 -0300 Subject: [PATCH 2/3] Expose EndpointType parameter in swift storage driver Signed-off-by: Cezar Sa Espinola --- registry/storage/driver/swift/swift.go | 2 ++ registry/storage/driver/swift/swift_test.go | 3 +++ 2 files changed, 5 insertions(+) diff --git a/registry/storage/driver/swift/swift.go b/registry/storage/driver/swift/swift.go index b72d0436..4191b8ba 100644 --- a/registry/storage/driver/swift/swift.go +++ b/registry/storage/driver/swift/swift.go @@ -72,6 +72,7 @@ type Parameters struct { AuthVersion int Container string Prefix string + EndpointType string InsecureSkipVerify bool ChunkSize int SecretKey string @@ -182,6 +183,7 @@ func New(params Parameters) (*Driver, error) { Domain: params.Domain, DomainId: params.DomainID, TrustId: params.TrustID, + EndpointType: swift.EndpointType(params.EndpointType), Transport: transport, ConnectTimeout: 60 * time.Second, Timeout: 15 * 60 * time.Second, diff --git a/registry/storage/driver/swift/swift_test.go b/registry/storage/driver/swift/swift_test.go index 655aa996..8979bd33 100644 --- a/registry/storage/driver/swift/swift_test.go +++ b/registry/storage/driver/swift/swift_test.go @@ -34,6 +34,7 @@ func init() { container string region string AuthVersion int + endpointType string insecureSkipVerify bool secretKey string accessKey string @@ -54,6 +55,7 @@ func init() { container = os.Getenv("SWIFT_CONTAINER_NAME") region = os.Getenv("SWIFT_REGION_NAME") AuthVersion, _ = strconv.Atoi(os.Getenv("SWIFT_AUTH_VERSION")) + endpointType = os.Getenv("SWIFT_ENDPOINT_TYPE") insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY")) secretKey = os.Getenv("SWIFT_SECRET_KEY") accessKey = os.Getenv("SWIFT_ACCESS_KEY") @@ -90,6 +92,7 @@ func init() { AuthVersion, container, root, + endpointType, insecureSkipVerify, defaultChunkSize, secretKey, From afb262bd5f7d975338169230ed02c825bccab3c7 Mon Sep 17 00:00:00 2001 From: Cezar Sa Espinola Date: Mon, 7 Mar 2016 18:52:53 -0300 Subject: [PATCH 3/3] Update swift driver docs removing dup table and adding endpointtype Signed-off-by: Cezar Sa Espinola --- docs/storage-drivers/swift.md | 161 +++------------------------------- 1 file changed, 10 insertions(+), 151 deletions(-) diff --git a/docs/storage-drivers/swift.md b/docs/storage-drivers/swift.md index 1555b181..b1a0c932 100644 --- a/docs/storage-drivers/swift.md +++ b/docs/storage-drivers/swift.md @@ -187,169 +187,28 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open The access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter. - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
- authurl - -

URL for obtaining an auth token.

-
- username - -

- Your OpenStack user name.

-

-
- password -

-

-

- Your OpenStack password. -

-
- container - -

- The name of your Swift container where you wish to store the registry's data. The driver creates the named container during its initialization. -

-
- tenant - -

- Optionally, your OpenStack tenant name. You can either use tenant or tenantid. -

-
- tenantid + authversion -

- Optionally, your OpenStack tenant id. You can either use tenant or tenantid. -

-
- domain + no -

- Optionally, your OpenStack domain name for Identity v3 API. You can either use domain or domainid. -

+ Specify the OpenStack Auth's version,for example 3. By default the driver will autodetect the auth's version from the AuthURL.
- domainid + endpointtype -

- Optionally, your OpenStack domain id for Identity v3 API. You can either use domain or domainid. -

-
- trustid + no -

- Optionally, your OpenStack trust id for Identity v3 API. -

+ The endpoint type used when connecting to swift. Possible values are `public`, `internal` and `admin`. Default is `public`.
- insecureskipverify - -

- Optionally, set insecureskipverify to true to skip TLS verification for your OpenStack provider. The driver uses false by default. -

-
- region - -

- Optionally, specify the OpenStack region name in which you would like to store objects (for example fr). -

-
- authversion - -

- Optionally, specify the OpenStack Auth's version,for example 3. By default the driver will autodetect the auth's version from the AuthURL. -

-
- chunksize - -

- Optionally, specify the segment size for Dynamic Large Objects uploads (performed by WriteStream) to Swift. The default is 5 MB. You might experience better performance for larger chunk sizes depending on the speed of your connection to Swift. -

-
- prefix - -

- Optionally, supply a prefix that will be applied to all Swift keys to allow you to segment data in your container if necessary. Defaults to the empty string which is the container's root.

-

-
- secretkey - -

- Optionally, the secret key used to generate temporary URLs.

-

-
- accesskey - -

- Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.

-

-
The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator