Add support for temporary URL for Swift driver
Signed-off-by: Sylvain Baubeau <sbaubeau@redhat.com>
This commit is contained in:
Sylvain Baubeau
parent
7759153f2f
commit
7c3281861f
3 changed files with 234 additions and 46 deletions
|
|
@ -93,6 +93,16 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
|
|||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>trustid</code>
|
||||
</td>
|
||||
<td>
|
||||
<p>
|
||||
Optionally, your OpenStack trust id for Identity v3 API.
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>insecureskipverify</code>
|
||||
|
|
@ -133,4 +143,58 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open
|
|||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>secretkey</code>
|
||||
</td>
|
||||
<td>
|
||||
<p>
|
||||
Optionally, the secret key used to generate temporary URLs.</p>
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>accesskey</code>
|
||||
</td>
|
||||
<td>
|
||||
<p>
|
||||
Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.</p>
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator
|
||||
disabled that feature, the configuration file can specify the following optional parameters :
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td>
|
||||
<code>tempurlcontainerkey</code>
|
||||
</td>
|
||||
<td>
|
||||
<p>
|
||||
Specify whether to use container secret key to generate temporary URL when set to true, or the account secret key otherwise.</p>
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<code>tempurlmethods</code>
|
||||
</td>
|
||||
<td>
|
||||
<p>
|
||||
Array of HTTP methods that are supported by the TempURL middleware of the Swift server. Example:</p>
|
||||
<code>
|
||||
- tempurlmethods:
|
||||
- GET
|
||||
- PUT
|
||||
- HEAD
|
||||
- POST
|
||||
- DELETE
|
||||
</code>
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -7,9 +7,6 @@
|
|||
// It supports both TempAuth authentication and Keystone authentication
|
||||
// (up to version 3).
|
||||
//
|
||||
// Since Swift has no concept of directories (directories are an abstration),
|
||||
// empty objects are created with the MIME type application/vnd.swift.directory.
|
||||
//
|
||||
// As Swift has a limit on the size of a single uploaded object (by default
|
||||
// this is 5GB), the driver makes use of the Swift Large Object Support
|
||||
// (http://docs.openstack.org/developer/swift/overview_large_objects.html).
|
||||
|
|
@ -24,12 +21,11 @@ import (
|
|||
"crypto/sha1"
|
||||
"crypto/tls"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
gopath "path"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -54,22 +50,34 @@ const minChunkSize = 1 << 20
|
|||
|
||||
// Parameters A struct that encapsulates all of the driver parameters after all values have been set
|
||||
type Parameters struct {
|
||||
Username string
|
||||
Password string
|
||||
AuthURL string
|
||||
Tenant string
|
||||
TenantID string
|
||||
Domain string
|
||||
DomainID string
|
||||
TrustID string
|
||||
Region string
|
||||
Container string
|
||||
Prefix string
|
||||
InsecureSkipVerify bool
|
||||
ChunkSize int
|
||||
Username string
|
||||
Password string
|
||||
AuthURL string
|
||||
Tenant string
|
||||
TenantID string
|
||||
Domain string
|
||||
DomainID string
|
||||
TrustID string
|
||||
Region string
|
||||
Container string
|
||||
Prefix string
|
||||
InsecureSkipVerify bool
|
||||
ChunkSize int
|
||||
SecretKey string
|
||||
AccessKey string
|
||||
TempURLContainerKey bool
|
||||
TempURLMethods []string
|
||||
}
|
||||
|
||||
type swiftInfo map[string]interface{}
|
||||
// swiftInfo maps the JSON structure returned by Swift /info endpoint
|
||||
type swiftInfo struct {
|
||||
Swift struct {
|
||||
Version string `mapstructure:"version"`
|
||||
}
|
||||
Tempurl struct {
|
||||
Methods []string `mapstructure:"methods"`
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
factory.Register(driverName, &swiftDriverFactory{})
|
||||
|
|
@ -83,11 +91,15 @@ func (factory *swiftDriverFactory) Create(parameters map[string]interface{}) (st
|
|||
}
|
||||
|
||||
type driver struct {
|
||||
Conn swift.Connection
|
||||
Container string
|
||||
Prefix string
|
||||
BulkDeleteSupport bool
|
||||
ChunkSize int
|
||||
Conn swift.Connection
|
||||
Container string
|
||||
Prefix string
|
||||
BulkDeleteSupport bool
|
||||
ChunkSize int
|
||||
SecretKey string
|
||||
AccessKey string
|
||||
TempURLContainerKey bool
|
||||
TempURLMethods []string
|
||||
}
|
||||
|
||||
type baseEmbed struct {
|
||||
|
|
@ -176,11 +188,65 @@ func New(params Parameters) (*Driver, error) {
|
|||
}
|
||||
|
||||
d := &driver{
|
||||
Conn: ct,
|
||||
Container: params.Container,
|
||||
Prefix: params.Prefix,
|
||||
BulkDeleteSupport: detectBulkDelete(params.AuthURL),
|
||||
ChunkSize: params.ChunkSize,
|
||||
Conn: ct,
|
||||
Container: params.Container,
|
||||
Prefix: params.Prefix,
|
||||
ChunkSize: params.ChunkSize,
|
||||
TempURLMethods: make([]string, 0),
|
||||
AccessKey: params.AccessKey,
|
||||
}
|
||||
|
||||
info := swiftInfo{}
|
||||
if config, err := d.Conn.QueryInfo(); err == nil {
|
||||
_, d.BulkDeleteSupport = config["bulk_delete"]
|
||||
|
||||
if err := mapstructure.Decode(config, &info); err == nil {
|
||||
d.TempURLContainerKey = info.Swift.Version >= "2.3.0"
|
||||
d.TempURLMethods = info.Tempurl.Methods
|
||||
}
|
||||
} else {
|
||||
d.TempURLContainerKey = params.TempURLContainerKey
|
||||
d.TempURLMethods = params.TempURLMethods
|
||||
}
|
||||
|
||||
if len(d.TempURLMethods) > 0 {
|
||||
secretKey := params.SecretKey
|
||||
if secretKey == "" {
|
||||
secretKey, _ = generateSecret()
|
||||
}
|
||||
|
||||
// Since Swift 2.2.2, we can now set secret keys on containers
|
||||
// in addition to the account secret keys. Use them in preference.
|
||||
if d.TempURLContainerKey {
|
||||
_, containerHeaders, err := d.Conn.Container(d.Container)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to fetch container info %s (%s)", d.Container, err)
|
||||
}
|
||||
|
||||
d.SecretKey = containerHeaders["X-Container-Meta-Temp-Url-Key"]
|
||||
if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
|
||||
m := swift.Metadata{}
|
||||
m["temp-url-key"] = secretKey
|
||||
if d.Conn.ContainerUpdate(d.Container, m.ContainerHeaders()); err == nil {
|
||||
d.SecretKey = secretKey
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Use the account secret key
|
||||
_, accountHeaders, err := d.Conn.Account()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to fetch account info (%s)", err)
|
||||
}
|
||||
|
||||
d.SecretKey = accountHeaders["X-Account-Meta-Temp-Url-Key"]
|
||||
if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) {
|
||||
m := swift.Metadata{}
|
||||
m["temp-url-key"] = secretKey
|
||||
if err := d.Conn.AccountUpdate(m.AccountHeaders()); err == nil {
|
||||
d.SecretKey = secretKey
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &Driver{
|
||||
|
|
@ -590,9 +656,58 @@ func (d *driver) Delete(ctx context.Context, path string) error {
|
|||
}
|
||||
|
||||
// URLFor returns a URL which may be used to retrieve the content stored at the given path.
|
||||
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
|
||||
func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
|
||||
return "", storagedriver.ErrUnsupportedMethod
|
||||
if d.SecretKey == "" {
|
||||
return "", storagedriver.ErrUnsupportedMethod
|
||||
}
|
||||
|
||||
methodString := "GET"
|
||||
method, ok := options["method"]
|
||||
if ok {
|
||||
if methodString, ok = method.(string); !ok {
|
||||
return "", storagedriver.ErrUnsupportedMethod
|
||||
}
|
||||
}
|
||||
|
||||
if methodString == "HEAD" {
|
||||
// A "HEAD" request on a temporary URL is allowed if the
|
||||
// signature was generated with "GET", "POST" or "PUT"
|
||||
methodString = "GET"
|
||||
}
|
||||
|
||||
supported := false
|
||||
for _, method := range d.TempURLMethods {
|
||||
if method == methodString {
|
||||
supported = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !supported {
|
||||
return "", storagedriver.ErrUnsupportedMethod
|
||||
}
|
||||
|
||||
expiresTime := time.Now().Add(20 * time.Minute)
|
||||
expires, ok := options["expiry"]
|
||||
if ok {
|
||||
et, ok := expires.(time.Time)
|
||||
if ok {
|
||||
expiresTime = et
|
||||
}
|
||||
}
|
||||
|
||||
tempURL := d.Conn.ObjectTempUrl(d.Container, d.swiftPath(path), d.SecretKey, methodString, expiresTime)
|
||||
|
||||
if d.AccessKey != "" {
|
||||
// On HP Cloud, the signature must be in the form of tenant_id:access_key:signature
|
||||
url, _ := url.Parse(tempURL)
|
||||
query := url.Query()
|
||||
query.Set("temp_url_sig", fmt.Sprintf("%s:%s:%s", d.Conn.TenantId, d.AccessKey, query.Get("temp_url_sig")))
|
||||
url.RawQuery = query.Encode()
|
||||
tempURL = url.String()
|
||||
}
|
||||
|
||||
return tempURL, nil
|
||||
}
|
||||
|
||||
func (d *driver) swiftPath(path string) string {
|
||||
|
|
@ -640,19 +755,6 @@ func (d *driver) createManifest(path string, segments string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func detectBulkDelete(authURL string) (bulkDelete bool) {
|
||||
resp, err := http.Get(gopath.Join(authURL, "..", "..") + "/info")
|
||||
if err == nil {
|
||||
defer resp.Body.Close()
|
||||
decoder := json.NewDecoder(resp.Body)
|
||||
var infos swiftInfo
|
||||
if decoder.Decode(&infos) == nil {
|
||||
_, bulkDelete = infos["bulk_delete"]
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func parseManifest(manifest string) (container string, prefix string) {
|
||||
components := strings.SplitN(manifest, "/", 2)
|
||||
container = components[0]
|
||||
|
|
@ -661,3 +763,11 @@ func parseManifest(manifest string) (container string, prefix string) {
|
|||
}
|
||||
return container, prefix
|
||||
}
|
||||
|
||||
func generateSecret() (string, error) {
|
||||
var secretBytes [32]byte
|
||||
if _, err := rand.Read(secretBytes[:]); err != nil {
|
||||
return "", fmt.Errorf("could not generate random bytes for Swift secret key: %v", err)
|
||||
}
|
||||
return hex.EncodeToString(secretBytes[:]), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"io/ioutil"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ncw/swift/swifttest"
|
||||
|
|
@ -33,8 +34,13 @@ func init() {
|
|||
container string
|
||||
region string
|
||||
insecureSkipVerify bool
|
||||
swiftServer *swifttest.SwiftServer
|
||||
err error
|
||||
secretKey string
|
||||
accessKey string
|
||||
containerKey bool
|
||||
tempURLMethods []string
|
||||
|
||||
swiftServer *swifttest.SwiftServer
|
||||
err error
|
||||
)
|
||||
username = os.Getenv("SWIFT_USERNAME")
|
||||
password = os.Getenv("SWIFT_PASSWORD")
|
||||
|
|
@ -47,6 +53,10 @@ func init() {
|
|||
container = os.Getenv("SWIFT_CONTAINER_NAME")
|
||||
region = os.Getenv("SWIFT_REGION_NAME")
|
||||
insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
|
||||
secretKey = os.Getenv("SWIFT_SECRET_KEY")
|
||||
accessKey = os.Getenv("SWIFT_ACCESS_KEY")
|
||||
containerKey, _ = strconv.ParseBool(os.Getenv("SWIFT_TEMPURL_CONTAINERKEY"))
|
||||
tempURLMethods = strings.Split(os.Getenv("SWIFT_TEMPURL_METHODS"), ",")
|
||||
|
||||
if username == "" || password == "" || authURL == "" || container == "" {
|
||||
if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil {
|
||||
|
|
@ -79,6 +89,10 @@ func init() {
|
|||
root,
|
||||
insecureSkipVerify,
|
||||
defaultChunkSize,
|
||||
secretKey,
|
||||
accessKey,
|
||||
containerKey,
|
||||
tempURLMethods,
|
||||
}
|
||||
|
||||
return New(parameters)
|
||||
|
|
|
|||
Loading…
Reference in a new issue