Add swift driver dependencies into Godep workspace.

Signed-off-by: Li Wenquan <wenquan.li@hp.com>
This commit is contained in:
davidli 2015-05-22 16:17:22 +08:00 committed by Sylvain Baubeau
parent 6dc1596be4
commit 8da60d6445
22 changed files with 5920 additions and 0 deletions

View file

@ -0,0 +1,4 @@
*~
*.pyc
test-env*
junk/

View file

@ -0,0 +1,10 @@
language: go
go:
- 1.1.2
- 1.2.2
- 1.3
- tip
script:
- go test

20
Godeps/_workspace/src/github.com/lebauce/swift/COPYING generated vendored Normal file
View file

@ -0,0 +1,20 @@
Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View file

@ -0,0 +1,106 @@
Swift
=====
This package provides an easy to use library for interfacing with
Swift / Openstack Object Storage / Rackspace cloud files from the Go
Language
See here for package docs
http://godoc.org/github.com/ncw/swift
[![Build Status](https://travis-ci.org/ncw/swift.png)](https://travis-ci.org/ncw/swift)
Install
-------
Use go to install the library
go get github.com/ncw/swift
Usage
-----
See here for full package docs
- http://godoc.org/github.com/ncw/swift
Here is a short example from the docs
import "github.com/ncw/swift"
// Create a connection
c := swift.Connection{
UserName: "user",
ApiKey: "key",
AuthUrl: "auth_url",
}
// Authenticate
err := c.Authenticate()
if err != nil {
panic(err)
}
// List all the containers
containers, err := c.ContainerNames(nil)
fmt.Println(containers)
// etc...
Additions
---------
The `rs` sub project contains a wrapper for the Rackspace specific CDN Management interface.
Testing
-------
To run the tests you can either use an embedded fake Swift server
either use a real Openstack Swift server or a Rackspace Cloud files account.
When using a real Swift server, you need to set these environment variables
before running the tests
export SWIFT_API_USER='user'
export SWIFT_API_KEY='key'
export SWIFT_AUTH_URL='https://url.of.auth.server/v1.0'
And optionally these if using v2 authentication
export SWIFT_TENANT='TenantName'
export SWIFT_TENANT_ID='TenantId'
Then run the tests with `go test`
License
-------
This is free software under the terms of MIT license (check COPYING file
included in this package).
Contact and support
-------------------
The project website is at:
- https://github.com/ncw/swift
There you can file bug reports, ask for help or contribute patches.
Authors
-------
- Nick Craig-Wood <nick@craig-wood.com>
Contributors
------------
- Brian "bojo" Jones <mojobojo@gmail.com>
- Janika Liiv <janika@toggl.com>
- Yamamoto, Hirotaka <ymmt2005@gmail.com>
- Stephen <yo@groks.org>
- platformpurple <stephen@platformpurple.com>
- Paul Querna <pquerna@apache.org>
- Livio Soares <liviobs@gmail.com>
- thesyncim <thesyncim@gmail.com>
- lsowen <lsowen@s1network.com>
- Sylvain Baubeau <sbaubeau@redhat.com>
- Chris Kastorff <encryptio@gmail.com>

279
Godeps/_workspace/src/github.com/lebauce/swift/auth.go generated vendored Normal file
View file

@ -0,0 +1,279 @@
package swift
import (
"bytes"
"encoding/json"
"net/http"
"net/url"
"strings"
)
// Auth defines the operations needed to authenticate with swift
//
// This encapsulates the different authentication schemes in use
type Authenticator interface {
Request(*Connection) (*http.Request, error)
Response(resp *http.Response) error
// The public storage URL - set Internal to true to read
// internal/service net URL
StorageUrl(Internal bool) string
// The access token
Token() string
// The CDN url if available
CdnUrl() string
}
// newAuth - create a new Authenticator from the AuthUrl
//
// A hint for AuthVersion can be provided
func newAuth(c *Connection) (Authenticator, error) {
AuthVersion := c.AuthVersion
if AuthVersion == 0 {
if strings.Contains(c.AuthUrl, "v2") {
AuthVersion = 2
} else if strings.Contains(c.AuthUrl, "v1") {
AuthVersion = 1
} else {
return nil, newErrorf(500, "Can't find AuthVersion in AuthUrl - set explicitly")
}
}
switch AuthVersion {
case 1:
return &v1Auth{}, nil
case 2:
return &v2Auth{
// Guess as to whether using API key or
// password it will try both eventually so
// this is just an optimization.
useApiKey: len(c.ApiKey) >= 32,
}, nil
}
return nil, newErrorf(500, "Auth Version %d not supported", AuthVersion)
}
// ------------------------------------------------------------
// v1 auth
type v1Auth struct {
Headers http.Header // V1 auth: the authentication headers so extensions can access them
}
// v1 Authentication - make request
func (auth *v1Auth) Request(c *Connection) (*http.Request, error) {
req, err := http.NewRequest("GET", c.AuthUrl, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", c.UserAgent)
req.Header.Set("X-Auth-Key", c.ApiKey)
req.Header.Set("X-Auth-User", c.UserName)
return req, nil
}
// v1 Authentication - read response
func (auth *v1Auth) Response(resp *http.Response) error {
auth.Headers = resp.Header
return nil
}
// v1 Authentication - read storage url
func (auth *v1Auth) StorageUrl(Internal bool) string {
storageUrl := auth.Headers.Get("X-Storage-Url")
if Internal {
newUrl, err := url.Parse(storageUrl)
if err != nil {
return storageUrl
}
newUrl.Host = "snet-" + newUrl.Host
storageUrl = newUrl.String()
}
return storageUrl
}
// v1 Authentication - read auth token
func (auth *v1Auth) Token() string {
return auth.Headers.Get("X-Auth-Token")
}
// v1 Authentication - read cdn url
func (auth *v1Auth) CdnUrl() string {
return auth.Headers.Get("X-CDN-Management-Url")
}
// ------------------------------------------------------------
// v2 Authentication
type v2Auth struct {
Auth *v2AuthResponse
Region string
useApiKey bool // if set will use API key not Password
useApiKeyOk bool // if set won't change useApiKey any more
notFirst bool // set after first run
}
// v2 Authentication - make request
func (auth *v2Auth) Request(c *Connection) (*http.Request, error) {
auth.Region = c.Region
// Toggle useApiKey if not first run and not OK yet
if auth.notFirst && !auth.useApiKeyOk {
auth.useApiKey = !auth.useApiKey
}
auth.notFirst = true
// Create a V2 auth request for the body of the connection
var v2i interface{}
if !auth.useApiKey {
// Normal swift authentication
v2 := v2AuthRequest{}
v2.Auth.PasswordCredentials.UserName = c.UserName
v2.Auth.PasswordCredentials.Password = c.ApiKey
v2.Auth.Tenant = c.Tenant
v2.Auth.TenantId = c.TenantId
v2i = v2
} else {
// Rackspace special with API Key
v2 := v2AuthRequestRackspace{}
v2.Auth.ApiKeyCredentials.UserName = c.UserName
v2.Auth.ApiKeyCredentials.ApiKey = c.ApiKey
v2.Auth.Tenant = c.Tenant
v2.Auth.TenantId = c.TenantId
v2i = v2
}
body, err := json.Marshal(v2i)
if err != nil {
return nil, err
}
url := c.AuthUrl
if !strings.HasSuffix(url, "/") {
url += "/"
}
url += "tokens"
req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
return req, nil
}
// v2 Authentication - read response
func (auth *v2Auth) Response(resp *http.Response) error {
auth.Auth = new(v2AuthResponse)
err := readJson(resp, auth.Auth)
// If successfully read Auth then no need to toggle useApiKey any more
if err == nil {
auth.useApiKeyOk = true
}
return err
}
// Finds the Endpoint Url of "type" from the v2AuthResponse using the
// Region if set or defaulting to the first one if not
//
// Returns "" if not found
func (auth *v2Auth) endpointUrl(Type string, Internal bool) string {
for _, catalog := range auth.Auth.Access.ServiceCatalog {
if catalog.Type == Type {
for _, endpoint := range catalog.Endpoints {
if auth.Region == "" || (auth.Region == endpoint.Region) {
if Internal {
return endpoint.InternalUrl
} else {
return endpoint.PublicUrl
}
}
}
}
}
return ""
}
// v2 Authentication - read storage url
//
// If Internal is true then it reads the private (internal / service
// net) URL.
func (auth *v2Auth) StorageUrl(Internal bool) string {
return auth.endpointUrl("object-store", Internal)
}
// v2 Authentication - read auth token
func (auth *v2Auth) Token() string {
return auth.Auth.Access.Token.Id
}
// v2 Authentication - read cdn url
func (auth *v2Auth) CdnUrl() string {
return auth.endpointUrl("rax:object-cdn", false)
}
// ------------------------------------------------------------
// V2 Authentication request
//
// http://docs.openstack.org/developer/keystone/api_curl_examples.html
// http://docs.rackspace.com/servers/api/v2/cs-gettingstarted/content/curl_auth.html
// http://docs.openstack.org/api/openstack-identity-service/2.0/content/POST_authenticate_v2.0_tokens_.html
type v2AuthRequest struct {
Auth struct {
PasswordCredentials struct {
UserName string `json:"username"`
Password string `json:"password"`
} `json:"passwordCredentials"`
Tenant string `json:"tenantName,omitempty"`
TenantId string `json:"tenantId,omitempty"`
} `json:"auth"`
}
// V2 Authentication request - Rackspace variant
//
// http://docs.openstack.org/developer/keystone/api_curl_examples.html
// http://docs.rackspace.com/servers/api/v2/cs-gettingstarted/content/curl_auth.html
// http://docs.openstack.org/api/openstack-identity-service/2.0/content/POST_authenticate_v2.0_tokens_.html
type v2AuthRequestRackspace struct {
Auth struct {
ApiKeyCredentials struct {
UserName string `json:"username"`
ApiKey string `json:"apiKey"`
} `json:"RAX-KSKEY:apiKeyCredentials"`
Tenant string `json:"tenantName,omitempty"`
TenantId string `json:"tenantId,omitempty"`
} `json:"auth"`
}
// V2 Authentication reply
//
// http://docs.openstack.org/developer/keystone/api_curl_examples.html
// http://docs.rackspace.com/servers/api/v2/cs-gettingstarted/content/curl_auth.html
// http://docs.openstack.org/api/openstack-identity-service/2.0/content/POST_authenticate_v2.0_tokens_.html
type v2AuthResponse struct {
Access struct {
ServiceCatalog []struct {
Endpoints []struct {
InternalUrl string
PublicUrl string
Region string
TenantId string
}
Name string
Type string
}
Token struct {
Expires string
Id string
Tenant struct {
Id string
Name string
}
}
User struct {
DefaultRegion string `json:"RAX-AUTH:defaultRegion"`
Id string
Name string
Roles []struct {
Description string
Id string
Name string
TenantId string
}
}
}
}

View file

@ -0,0 +1,28 @@
// Go 1.0 compatibility functions
// +build !go1.1
package swift
import (
"log"
"net/http"
"time"
)
// Cancel the request - doesn't work under < go 1.1
func cancelRequest(transport http.RoundTripper, req *http.Request) {
log.Printf("Tried to cancel a request but couldn't - recompile with go 1.1")
}
// Reset a timer - Doesn't work properly < go 1.1
//
// This is quite hard to do properly under go < 1.1 so we do a crude
// approximation and hope that everyone upgrades to go 1.1 quickly
func resetTimer(t *time.Timer, d time.Duration) {
t.Stop()
// Very likely this doesn't actually work if we are already
// selecting on t.C. However we've stopped the original timer
// so won't break transfers but may not time them out :-(
*t = *time.NewTimer(d)
}

View file

@ -0,0 +1,24 @@
// Go 1.1 and later compatibility functions
//
// +build go1.1
package swift
import (
"net/http"
"time"
)
// Cancel the request
func cancelRequest(transport http.RoundTripper, req *http.Request) {
if tr, ok := transport.(interface {
CancelRequest(*http.Request)
}); ok {
tr.CancelRequest(req)
}
}
// Reset a timer
func resetTimer(t *time.Timer, d time.Duration) {
t.Reset(d)
}

19
Godeps/_workspace/src/github.com/lebauce/swift/doc.go generated vendored Normal file
View file

@ -0,0 +1,19 @@
/*
Package swift provides an easy to use interface to Swift / Openstack Object Storage / Rackspace Cloud Files
Standard Usage
Most of the work is done through the Container*() and Object*() methods.
All methods are safe to use concurrently in multiple go routines.
Object Versioning
As defined by http://docs.openstack.org/api/openstack-object-storage/1.0/content/Object_Versioning-e1e3230.html#d6e983 one can create a container which allows for version control of files. The suggested method is to create a version container for holding all non-current files, and a current container for holding the latest version that the file points to. The container and objects inside it can be used in the standard manner, however, pushing a file multiple times will result in it being copied to the version container and the new file put in it's place. If the current file is deleted, the previous file in the version container will replace it. This means that if a file is updated 5 times, it must be deleted 5 times to be completely removed from the system.
Rackspace Sub Module
This module specifically allows the enabling/disabling of Rackspace Cloud File CDN management on a container. This is specific to the Rackspace API and not Swift/Openstack, therefore it has been placed in a submodule. One can easily create a RsConnection and use it like the standard Connection to access and manipulate containers and objects.
*/
package swift

View file

@ -0,0 +1,97 @@
// Copyright...
// This example demonstrates opening a Connection and doing some basic operations.
package swift_test
import (
"fmt"
"github.com/ncw/swift"
)
func ExampleConnection() {
// Create a v1 auth connection
c := swift.Connection{
// This should be your username
UserName: "user",
// This should be your api key
ApiKey: "key",
// This should be a v1 auth url, eg
// Rackspace US https://auth.api.rackspacecloud.com/v1.0
// Rackspace UK https://lon.auth.api.rackspacecloud.com/v1.0
// Memset Memstore UK https://auth.storage.memset.com/v1.0
AuthUrl: "auth_url",
}
// Authenticate
err := c.Authenticate()
if err != nil {
panic(err)
}
// List all the containers
containers, err := c.ContainerNames(nil)
fmt.Println(containers)
// etc...
// ------ or alternatively create a v2 connection ------
// Create a v2 auth connection
c = swift.Connection{
// This is the sub user for the storage - eg "admin"
UserName: "user",
// This should be your api key
ApiKey: "key",
// This should be a version2 auth url, eg
// Rackspace v2 https://identity.api.rackspacecloud.com/v2.0
// Memset Memstore v2 https://auth.storage.memset.com/v2.0
AuthUrl: "v2_auth_url",
// Region to use - default is use first region if unset
Region: "LON",
// Name of the tenant - this is likely your username
Tenant: "jim",
}
// as above...
}
var container string
func ExampleConnection_ObjectsWalk() {
objects := make([]string, 0)
err := c.ObjectsWalk(container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
newObjects, err := c.ObjectNames(container, opts)
if err == nil {
objects = append(objects, newObjects...)
}
return newObjects, err
})
fmt.Println("Found all the objects", objects, err)
}
func ExampleConnection_VersionContainerCreate() {
// Use the helper method to create the current and versions container.
if err := c.VersionContainerCreate("cds", "cd-versions"); err != nil {
fmt.Print(err.Error())
}
}
func ExampleConnection_VersionEnable() {
// Build the containers manually and enable them.
if err := c.ContainerCreate("movie-versions", nil); err != nil {
fmt.Print(err.Error())
}
if err := c.ContainerCreate("movies", nil); err != nil {
fmt.Print(err.Error())
}
if err := c.VersionEnable("movies", "movie-versions"); err != nil {
fmt.Print(err.Error())
}
// Access the primary container as usual with ObjectCreate(), ObjectPut(), etc.
// etc...
}
func ExampleConnection_VersionDisable() {
// Disable versioning on a container. Note that this does not delete the versioning container.
c.VersionDisable("movies")
}

174
Godeps/_workspace/src/github.com/lebauce/swift/meta.go generated vendored Normal file
View file

@ -0,0 +1,174 @@
// Metadata manipulation in and out of Headers
package swift
import (
"fmt"
"net/http"
"strconv"
"strings"
"time"
)
// Metadata stores account, container or object metadata.
type Metadata map[string]string
// Metadata gets the Metadata starting with the metaPrefix out of the Headers.
//
// The keys in the Metadata will be converted to lower case
func (h Headers) Metadata(metaPrefix string) Metadata {
m := Metadata{}
metaPrefix = http.CanonicalHeaderKey(metaPrefix)
for key, value := range h {
if strings.HasPrefix(key, metaPrefix) {
metaKey := strings.ToLower(key[len(metaPrefix):])
m[metaKey] = value
}
}
return m
}
// AccountMetadata converts Headers from account to a Metadata.
//
// The keys in the Metadata will be converted to lower case.
func (h Headers) AccountMetadata() Metadata {
return h.Metadata("X-Account-Meta-")
}
// ContainerMetadata converts Headers from container to a Metadata.
//
// The keys in the Metadata will be converted to lower case.
func (h Headers) ContainerMetadata() Metadata {
return h.Metadata("X-Container-Meta-")
}
// ObjectMetadata converts Headers from object to a Metadata.
//
// The keys in the Metadata will be converted to lower case.
func (h Headers) ObjectMetadata() Metadata {
return h.Metadata("X-Object-Meta-")
}
// Headers convert the Metadata starting with the metaPrefix into a
// Headers.
//
// The keys in the Metadata will be converted from lower case to http
// Canonical (see http.CanonicalHeaderKey).
func (m Metadata) Headers(metaPrefix string) Headers {
h := Headers{}
for key, value := range m {
key = http.CanonicalHeaderKey(metaPrefix + key)
h[key] = value
}
return h
}
// AccountHeaders converts the Metadata for the account.
func (m Metadata) AccountHeaders() Headers {
return m.Headers("X-Account-Meta-")
}
// ContainerHeaders converts the Metadata for the container.
func (m Metadata) ContainerHeaders() Headers {
return m.Headers("X-Container-Meta-")
}
// ObjectHeaders converts the Metadata for the object.
func (m Metadata) ObjectHeaders() Headers {
return m.Headers("X-Object-Meta-")
}
// Turns a number of ns into a floating point string in seconds
//
// Trims trailing zeros and guaranteed to be perfectly accurate
func nsToFloatString(ns int64) string {
if ns < 0 {
return "-" + nsToFloatString(-ns)
}
result := fmt.Sprintf("%010d", ns)
split := len(result) - 9
result, decimals := result[:split], result[split:]
decimals = strings.TrimRight(decimals, "0")
if decimals != "" {
result += "."
result += decimals
}
return result
}
// Turns a floating point string in seconds into a ns integer
//
// Guaranteed to be perfectly accurate
func floatStringToNs(s string) (int64, error) {
const zeros = "000000000"
if point := strings.IndexRune(s, '.'); point >= 0 {
tail := s[point+1:]
if fill := 9 - len(tail); fill < 0 {
tail = tail[:9]
} else {
tail += zeros[:fill]
}
s = s[:point] + tail
} else if len(s) > 0 { // Make sure empty string produces an error
s += zeros
}
return strconv.ParseInt(s, 10, 64)
}
// FloatStringToTime converts a floating point number string to a time.Time
//
// The string is floating point number of seconds since the epoch
// (Unix time). The number should be in fixed point format (not
// exponential), eg "1354040105.123456789" which represents the time
// "2012-11-27T18:15:05.123456789Z"
//
// Some care is taken to preserve all the accuracy in the time.Time
// (which wouldn't happen with a naive conversion through float64) so
// a round trip conversion won't change the data.
//
// If an error is returned then time will be returned as the zero time.
func FloatStringToTime(s string) (t time.Time, err error) {
ns, err := floatStringToNs(s)
if err != nil {
return
}
t = time.Unix(0, ns)
return
}
// TimeToFloatString converts a time.Time object to a floating point string
//
// The string is floating point number of seconds since the epoch
// (Unix time). The number is in fixed point format (not
// exponential), eg "1354040105.123456789" which represents the time
// "2012-11-27T18:15:05.123456789Z". Trailing zeros will be dropped
// from the output.
//
// Some care is taken to preserve all the accuracy in the time.Time
// (which wouldn't happen with a naive conversion through float64) so
// a round trip conversion won't change the data.
func TimeToFloatString(t time.Time) string {
return nsToFloatString(t.UnixNano())
}
// Read a modification time (mtime) from a Metadata object
//
// This is a defacto standard (used in the official python-swiftclient
// amongst others) for storing the modification time (as read using
// os.Stat) for an object. It is stored using the key 'mtime', which
// for example when written to an object will be 'X-Object-Meta-Mtime'.
//
// If an error is returned then time will be returned as the zero time.
func (m Metadata) GetModTime() (t time.Time, err error) {
return FloatStringToTime(m["mtime"])
}
// Write an modification time (mtime) to a Metadata object
//
// This is a defacto standard (used in the official python-swiftclient
// amongst others) for storing the modification time (as read using
// os.Stat) for an object. It is stored using the key 'mtime', which
// for example when written to an object will be 'X-Object-Meta-Mtime'.
func (m Metadata) SetModTime(t time.Time) {
m["mtime"] = TimeToFloatString(t)
}

View file

@ -0,0 +1,213 @@
// Tests for swift metadata
package swift
import (
"testing"
"time"
)
func TestHeadersToMetadata(t *testing.T) {
}
func TestHeadersToAccountMetadata(t *testing.T) {
}
func TestHeadersToContainerMetadata(t *testing.T) {
}
func TestHeadersToObjectMetadata(t *testing.T) {
}
func TestMetadataToHeaders(t *testing.T) {
}
func TestMetadataToAccountHeaders(t *testing.T) {
}
func TestMetadataToContainerHeaders(t *testing.T) {
}
func TestMetadataToObjectHeaders(t *testing.T) {
}
func TestNsToFloatString(t *testing.T) {
for _, d := range []struct {
ns int64
fs string
}{
{0, "0"},
{1, "0.000000001"},
{1000, "0.000001"},
{1000000, "0.001"},
{100000000, "0.1"},
{1000000000, "1"},
{10000000000, "10"},
{12345678912, "12.345678912"},
{12345678910, "12.34567891"},
{12345678900, "12.3456789"},
{12345678000, "12.345678"},
{12345670000, "12.34567"},
{12345600000, "12.3456"},
{12345000000, "12.345"},
{12340000000, "12.34"},
{12300000000, "12.3"},
{12000000000, "12"},
{10000000000, "10"},
{1347717491123123123, "1347717491.123123123"},
} {
if nsToFloatString(d.ns) != d.fs {
t.Error("Failed", d.ns, "!=", d.fs)
}
if d.ns > 0 && nsToFloatString(-d.ns) != "-"+d.fs {
t.Error("Failed on negative", d.ns, "!=", d.fs)
}
}
}
func TestFloatStringToNs(t *testing.T) {
for _, d := range []struct {
ns int64
fs string
}{
{0, "0"},
{0, "0."},
{0, ".0"},
{0, "0.0"},
{0, "0.0000000001"},
{1, "0.000000001"},
{1000, "0.000001"},
{1000000, "0.001"},
{100000000, "0.1"},
{100000000, "0.10"},
{100000000, "0.1000000001"},
{1000000000, "1"},
{1000000000, "1."},
{1000000000, "1.0"},
{10000000000, "10"},
{12345678912, "12.345678912"},
{12345678912, "12.3456789129"},
{12345678912, "12.34567891299"},
{12345678910, "12.34567891"},
{12345678900, "12.3456789"},
{12345678000, "12.345678"},
{12345670000, "12.34567"},
{12345600000, "12.3456"},
{12345000000, "12.345"},
{12340000000, "12.34"},
{12300000000, "12.3"},
{12000000000, "12"},
{10000000000, "10"},
// This is a typical value which has more bits in than a float64
{1347717491123123123, "1347717491.123123123"},
} {
ns, err := floatStringToNs(d.fs)
if err != nil {
t.Error("Failed conversion", err)
}
if ns != d.ns {
t.Error("Failed", d.fs, "!=", d.ns, "was", ns)
}
if d.ns > 0 {
ns, err := floatStringToNs("-" + d.fs)
if err != nil {
t.Error("Failed conversion", err)
}
if ns != -d.ns {
t.Error("Failed on negative", -d.ns, "!=", "-"+d.fs)
}
}
}
// These are expected to produce errors
for _, fs := range []string{
"",
" 1",
"- 1",
"- 1",
"1.-1",
"1.0.0",
"1x0",
} {
ns, err := floatStringToNs(fs)
if err == nil {
t.Error("Didn't produce expected error", fs, ns)
}
}
}
func TestGetModTime(t *testing.T) {
for _, d := range []struct {
ns string
t string
}{
{"1354040105", "2012-11-27T18:15:05Z"},
{"1354040105.", "2012-11-27T18:15:05Z"},
{"1354040105.0", "2012-11-27T18:15:05Z"},
{"1354040105.000000000000", "2012-11-27T18:15:05Z"},
{"1354040105.123", "2012-11-27T18:15:05.123Z"},
{"1354040105.123456", "2012-11-27T18:15:05.123456Z"},
{"1354040105.123456789", "2012-11-27T18:15:05.123456789Z"},
{"1354040105.123456789123", "2012-11-27T18:15:05.123456789Z"},
{"0", "1970-01-01T00:00:00.000000000Z"},
} {
expected, err := time.Parse(time.RFC3339, d.t)
if err != nil {
t.Error("Bad test", err)
}
m := Metadata{"mtime": d.ns}
actual, err := m.GetModTime()
if err != nil {
t.Error("Parse error", err)
}
if !actual.Equal(expected) {
t.Error("Expecting", expected, expected.UnixNano(), "got", actual, actual.UnixNano())
}
}
for _, ns := range []string{
"EMPTY",
"",
" 1",
"- 1",
"- 1",
"1.-1",
"1.0.0",
"1x0",
} {
m := Metadata{}
if ns != "EMPTY" {
m["mtime"] = ns
}
actual, err := m.GetModTime()
if err == nil {
t.Error("Expected error not produced")
}
if !actual.IsZero() {
t.Error("Expected output to be zero")
}
}
}
func TestSetModTime(t *testing.T) {
for _, d := range []struct {
ns string
t string
}{
{"1354040105", "2012-11-27T18:15:05Z"},
{"1354040105", "2012-11-27T18:15:05.000000Z"},
{"1354040105.123", "2012-11-27T18:15:05.123Z"},
{"1354040105.123456", "2012-11-27T18:15:05.123456Z"},
{"1354040105.123456789", "2012-11-27T18:15:05.123456789Z"},
{"0", "1970-01-01T00:00:00.000000000Z"},
} {
time, err := time.Parse(time.RFC3339, d.t)
if err != nil {
t.Error("Bad test", err)
}
m := Metadata{}
m.SetModTime(time)
if m["mtime"] != d.ns {
t.Error("mtime wrong", m, "should be", d.ns)
}
}
}

View file

@ -0,0 +1,55 @@
Notes on Go Swift
=================
Make a builder style interface like the Google Go APIs? Advantages
are that it is easy to add named methods to the service object to do
specific things. Slightly less efficient. Not sure about how to
return extra stuff though - in an object?
Make a container struct so these could be methods on it?
Make noResponse check for 204?
Make storage public so it can be extended easily?
Rename to go-swift to match user agent string?
Reconnect on auth error - 401 when token expires isn't tested
Make more api compatible with python cloudfiles?
Retry operations on timeout / network errors?
- also 408 error
- GET requests only?
Make Connection thread safe - whenever it is changed take a write lock whenever it is read from a read lock
Add extra headers field to Connection (for via etc)
Make errors use an error heirachy then can catch them with a type assertion
Error(...)
ObjectCorrupted{ Error }
Make a Debug flag in connection for logging stuff
Object If-Match, If-None-Match, If-Modified-Since, If-Unmodified-Since etc
Object range
Object create, update with X-Delete-At or X-Delete-After
Large object support
- check uploads are less than 5GB in normal mode?
Access control CORS?
Swift client retries and backs off for all types of errors
Implement net error interface?
type Error interface {
error
Timeout() bool // Is the error a timeout?
Temporary() bool // Is the error temporary?
}

View file

@ -0,0 +1,83 @@
package rs
import (
"errors"
"net/http"
"strconv"
"github.com/ncw/swift"
)
// RsConnection is a RackSpace specific wrapper to the core swift library which
// exposes the RackSpace CDN commands via the CDN Management URL interface.
type RsConnection struct {
swift.Connection
cdnUrl string
}
// manage is similar to the swift storage method, but uses the CDN Management URL for CDN specific calls.
func (c *RsConnection) manage(p swift.RequestOpts) (resp *http.Response, headers swift.Headers, err error) {
p.OnReAuth = func() (string, error) {
if c.cdnUrl == "" {
c.cdnUrl = c.Auth.CdnUrl()
}
if c.cdnUrl == "" {
return "", errors.New("The X-CDN-Management-Url does not exist on the authenticated platform")
}
return c.cdnUrl, nil
}
if c.Authenticated() {
_, err = p.OnReAuth()
if err != nil {
return nil, nil, err
}
}
return c.Connection.Call(c.cdnUrl, p)
}
// ContainerCDNEnable enables a container for public CDN usage.
//
// Change the default TTL of 259200 seconds (72 hours) by passing in an integer value.
//
// This method can be called again to change the TTL.
func (c *RsConnection) ContainerCDNEnable(container string, ttl int) (swift.Headers, error) {
h := swift.Headers{"X-CDN-Enabled": "true"}
if ttl > 0 {
h["X-TTL"] = strconv.Itoa(ttl)
}
_, headers, err := c.manage(swift.RequestOpts{
Container: container,
Operation: "PUT",
ErrorMap: swift.ContainerErrorMap,
NoResponse: true,
Headers: h,
})
return headers, err
}
// ContainerCDNDisable disables CDN access to a container.
func (c *RsConnection) ContainerCDNDisable(container string) error {
h := swift.Headers{"X-CDN-Enabled": "false"}
_, _, err := c.manage(swift.RequestOpts{
Container: container,
Operation: "PUT",
ErrorMap: swift.ContainerErrorMap,
NoResponse: true,
Headers: h,
})
return err
}
// ContainerCDNMeta returns the CDN metadata for a container.
func (c *RsConnection) ContainerCDNMeta(container string) (swift.Headers, error) {
_, headers, err := c.manage(swift.RequestOpts{
Container: container,
Operation: "HEAD",
ErrorMap: swift.ContainerErrorMap,
NoResponse: true,
Headers: swift.Headers{},
})
return headers, err
}

View file

@ -0,0 +1,96 @@
// See swift_test.go for requirements to run this test.
package rs_test
import (
"os"
"testing"
"github.com/ncw/swift/rs"
)
var (
c rs.RsConnection
)
const (
CONTAINER = "GoSwiftUnitTest"
OBJECT = "test_object"
CONTENTS = "12345"
CONTENT_SIZE = int64(len(CONTENTS))
CONTENT_MD5 = "827ccb0eea8a706c4c34a16891f84e7b"
)
// Test functions are run in order - this one must be first!
func TestAuthenticate(t *testing.T) {
UserName := os.Getenv("SWIFT_API_USER")
ApiKey := os.Getenv("SWIFT_API_KEY")
AuthUrl := os.Getenv("SWIFT_AUTH_URL")
if UserName == "" || ApiKey == "" || AuthUrl == "" {
t.Fatal("SWIFT_API_USER, SWIFT_API_KEY and SWIFT_AUTH_URL not all set")
}
c = rs.RsConnection{}
c.UserName = UserName
c.ApiKey = ApiKey
c.AuthUrl = AuthUrl
err := c.Authenticate()
if err != nil {
t.Fatal("Auth failed", err)
}
if !c.Authenticated() {
t.Fatal("Not authenticated")
}
}
// Setup
func TestContainerCreate(t *testing.T) {
err := c.ContainerCreate(CONTAINER, nil)
if err != nil {
t.Fatal(err)
}
}
func TestCDNEnable(t *testing.T) {
headers, err := c.ContainerCDNEnable(CONTAINER, 0)
if err != nil {
t.Error(err)
}
if _, ok := headers["X-Cdn-Uri"]; !ok {
t.Error("Failed to enable CDN for container")
}
}
func TestOnReAuth(t *testing.T) {
c2 := rs.RsConnection{}
c2.UserName = c.UserName
c2.ApiKey = c.ApiKey
c2.AuthUrl = c.AuthUrl
_, err := c2.ContainerCDNEnable(CONTAINER, 0)
if err != nil {
t.Fatalf("Failed to reauthenticate: %v", err)
}
}
func TestCDNMeta(t *testing.T) {
headers, err := c.ContainerCDNMeta(CONTAINER)
if err != nil {
t.Error(err)
}
if _, ok := headers["X-Cdn-Uri"]; !ok {
t.Error("CDN is not enabled")
}
}
func TestCDNDisable(t *testing.T) {
err := c.ContainerCDNDisable(CONTAINER) // files stick in CDN until TTL expires
if err != nil {
t.Error(err)
}
}
// Teardown
func TestContainerDelete(t *testing.T) {
err := c.ContainerDelete(CONTAINER)
if err != nil {
t.Fatal(err)
}
}

1848
Godeps/_workspace/src/github.com/lebauce/swift/swift.go generated vendored Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,409 @@
// This tests the swift package internals
//
// It does not require access to a swift server
//
// FIXME need to add more tests and to check URLs and parameters
package swift
import (
"fmt"
"io"
"net"
"net/http"
"testing"
// "net/http/httputil"
// "os"
)
const (
TEST_ADDRESS = "localhost:5324"
AUTH_URL = "http://" + TEST_ADDRESS + "/v1.0"
PROXY_URL = "http://" + TEST_ADDRESS + "/proxy"
USERNAME = "test"
APIKEY = "apikey"
AUTH_TOKEN = "token"
)
// Globals
var (
server *SwiftServer
c *Connection
)
// SwiftServer implements a test swift server
type SwiftServer struct {
t *testing.T
checks []*Check
}
// Used to check and reply to http transactions
type Check struct {
in Headers
out Headers
rx *string
tx *string
err *Error
url *string
}
// Add a in check
func (check *Check) In(in Headers) *Check {
check.in = in
return check
}
// Add an out check
func (check *Check) Out(out Headers) *Check {
check.out = out
return check
}
// Add an Error check
func (check *Check) Error(StatusCode int, Text string) *Check {
check.err = newError(StatusCode, Text)
return check
}
// Add a rx check
func (check *Check) Rx(rx string) *Check {
check.rx = &rx
return check
}
// Add an tx check
func (check *Check) Tx(tx string) *Check {
check.tx = &tx
return check
}
// Add an URL check
func (check *Check) Url(url string) *Check {
check.url = &url
return check
}
// Add a check
func (s *SwiftServer) AddCheck(t *testing.T) *Check {
server.t = t
check := &Check{
in: Headers{},
out: Headers{},
err: nil,
}
s.checks = append(s.checks, check)
return check
}
// Responds to a request
func (s *SwiftServer) Respond(w http.ResponseWriter, r *http.Request) {
if len(s.checks) < 1 {
s.t.Fatal("Unexpected http transaction")
}
check := s.checks[0]
s.checks = s.checks[1:]
// Check URL
if check.url != nil && *check.url != r.URL.String() {
s.t.Errorf("Expecting URL %q but got %q", *check.url, r.URL)
}
// Check headers
for k, v := range check.in {
actual := r.Header.Get(k)
if actual != v {
s.t.Errorf("Expecting header %q=%q but got %q", k, v, actual)
}
}
// Write output headers
h := w.Header()
for k, v := range check.out {
h.Set(k, v)
}
// Return an error if required
if check.err != nil {
http.Error(w, check.err.Text, check.err.StatusCode)
} else {
if check.tx != nil {
_, err := w.Write([]byte(*check.tx))
if err != nil {
s.t.Error("Write failed", err)
}
}
}
}
// Checks to see all responses are used up
func (s *SwiftServer) Finished() {
if len(s.checks) > 0 {
s.t.Error("Unused checks", s.checks)
}
}
func handle(w http.ResponseWriter, r *http.Request) {
// out, _ := httputil.DumpRequest(r, true)
// os.Stdout.Write(out)
server.Respond(w, r)
}
func NewSwiftServer() *SwiftServer {
server := &SwiftServer{}
http.HandleFunc("/", handle)
go http.ListenAndServe(TEST_ADDRESS, nil)
fmt.Print("Waiting for server to start ")
for {
fmt.Print(".")
conn, err := net.Dial("tcp", TEST_ADDRESS)
if err == nil {
conn.Close()
fmt.Println(" Started")
break
}
}
return server
}
func init() {
server = NewSwiftServer()
c = &Connection{
UserName: USERNAME,
ApiKey: APIKEY,
AuthUrl: AUTH_URL,
}
}
// Check the error is a swift error
func checkError(t *testing.T, err error, StatusCode int, Text string) {
if err == nil {
t.Fatal("No error returned")
}
err2, ok := err.(*Error)
if !ok {
t.Fatal("Bad error type")
}
if err2.StatusCode != StatusCode {
t.Fatalf("Bad status code, expecting %d got %d", StatusCode, err2.StatusCode)
}
if err2.Text != Text {
t.Fatalf("Bad error string, expecting %q got %q", Text, err2.Text)
}
}
// FIXME copied from swift_test.go
func compareMaps(t *testing.T, a, b map[string]string) {
if len(a) != len(b) {
t.Error("Maps different sizes", a, b)
}
for ka, va := range a {
if vb, ok := b[ka]; !ok || va != vb {
t.Error("Difference in key", ka, va, b[ka])
}
}
for kb, vb := range b {
if va, ok := a[kb]; !ok || vb != va {
t.Error("Difference in key", kb, vb, a[kb])
}
}
}
func TestInternalError(t *testing.T) {
e := newError(404, "Not Found!")
if e.StatusCode != 404 || e.Text != "Not Found!" {
t.Fatal("Bad error")
}
if e.Error() != "Not Found!" {
t.Fatal("Bad error")
}
}
func testCheckClose(c io.Closer, e error) (err error) {
err = e
defer checkClose(c, &err)
return
}
// Make a closer which returns the error of our choice
type myCloser struct {
err error
}
func (c *myCloser) Close() error {
return c.err
}
func TestInternalCheckClose(t *testing.T) {
if testCheckClose(&myCloser{nil}, nil) != nil {
t.Fatal("bad 1")
}
if testCheckClose(&myCloser{nil}, ObjectCorrupted) != ObjectCorrupted {
t.Fatal("bad 2")
}
if testCheckClose(&myCloser{ObjectNotFound}, nil) != ObjectNotFound {
t.Fatal("bad 3")
}
if testCheckClose(&myCloser{ObjectNotFound}, ObjectCorrupted) != ObjectCorrupted {
t.Fatal("bad 4")
}
}
func TestInternalParseHeaders(t *testing.T) {
resp := &http.Response{StatusCode: 200}
if c.parseHeaders(resp, nil) != nil {
t.Error("Bad 1")
}
if c.parseHeaders(resp, authErrorMap) != nil {
t.Error("Bad 1")
}
resp = &http.Response{StatusCode: 299}
if c.parseHeaders(resp, nil) != nil {
t.Error("Bad 1")
}
resp = &http.Response{StatusCode: 199, Status: "BOOM"}
checkError(t, c.parseHeaders(resp, nil), 199, "HTTP Error: 199: BOOM")
resp = &http.Response{StatusCode: 300, Status: "BOOM"}
checkError(t, c.parseHeaders(resp, nil), 300, "HTTP Error: 300: BOOM")
resp = &http.Response{StatusCode: 404, Status: "BOOM"}
checkError(t, c.parseHeaders(resp, nil), 404, "HTTP Error: 404: BOOM")
if c.parseHeaders(resp, ContainerErrorMap) != ContainerNotFound {
t.Error("Bad 1")
}
if c.parseHeaders(resp, objectErrorMap) != ObjectNotFound {
t.Error("Bad 1")
}
}
func TestInternalReadHeaders(t *testing.T) {
resp := &http.Response{Header: http.Header{}}
compareMaps(t, readHeaders(resp), Headers{})
resp = &http.Response{Header: http.Header{
"one": []string{"1"},
"two": []string{"2"},
}}
compareMaps(t, readHeaders(resp), Headers{"one": "1", "two": "2"})
// FIXME this outputs a log which we should test and check
resp = &http.Response{Header: http.Header{
"one": []string{"1", "11", "111"},
"two": []string{"2"},
}}
compareMaps(t, readHeaders(resp), Headers{"one": "1", "two": "2"})
}
func TestInternalStorage(t *testing.T) {
// FIXME
}
// ------------------------------------------------------------
func TestInternalAuthenticate(t *testing.T) {
server.AddCheck(t).In(Headers{
"User-Agent": DefaultUserAgent,
"X-Auth-Key": APIKEY,
"X-Auth-User": USERNAME,
}).Out(Headers{
"X-Storage-Url": PROXY_URL,
"X-Auth-Token": AUTH_TOKEN,
}).Url("/v1.0")
defer server.Finished()
err := c.Authenticate()
if err != nil {
t.Fatal(err)
}
if c.StorageUrl != PROXY_URL {
t.Error("Bad storage url")
}
if c.AuthToken != AUTH_TOKEN {
t.Error("Bad auth token")
}
if !c.Authenticated() {
t.Error("Didn't authenticate")
}
}
func TestInternalAuthenticateDenied(t *testing.T) {
server.AddCheck(t).Error(400, "Bad request")
server.AddCheck(t).Error(401, "DENIED")
defer server.Finished()
c.UnAuthenticate()
err := c.Authenticate()
if err != AuthorizationFailed {
t.Fatal("Expecting AuthorizationFailed", err)
}
// FIXME
// if c.Authenticated() {
// t.Fatal("Expecting not authenticated")
// }
}
func TestInternalAuthenticateBad(t *testing.T) {
server.AddCheck(t).Out(Headers{
"X-Storage-Url": PROXY_URL,
})
defer server.Finished()
err := c.Authenticate()
checkError(t, err, 0, "Response didn't have storage url and auth token")
if c.Authenticated() {
t.Fatal("Expecting not authenticated")
}
server.AddCheck(t).Out(Headers{
"X-Auth-Token": AUTH_TOKEN,
})
err = c.Authenticate()
checkError(t, err, 0, "Response didn't have storage url and auth token")
if c.Authenticated() {
t.Fatal("Expecting not authenticated")
}
server.AddCheck(t)
err = c.Authenticate()
checkError(t, err, 0, "Response didn't have storage url and auth token")
if c.Authenticated() {
t.Fatal("Expecting not authenticated")
}
server.AddCheck(t).Out(Headers{
"X-Storage-Url": PROXY_URL,
"X-Auth-Token": AUTH_TOKEN,
})
err = c.Authenticate()
if err != nil {
t.Fatal(err)
}
if !c.Authenticated() {
t.Fatal("Expecting authenticated")
}
}
func testContainerNames(t *testing.T, rx string, expected []string) {
server.AddCheck(t).In(Headers{
"User-Agent": DefaultUserAgent,
"X-Auth-Token": AUTH_TOKEN,
}).Tx(rx).Url("/proxy")
containers, err := c.ContainerNames(nil)
if err != nil {
t.Fatal(err)
}
if len(containers) != len(expected) {
t.Fatal("Wrong number of containers", len(containers), rx, len(expected), expected)
}
for i := range containers {
if containers[i] != expected[i] {
t.Error("Bad container", containers[i], expected[i])
}
}
}
func TestInternalContainerNames(t *testing.T) {
defer server.Finished()
testContainerNames(t, "", []string{})
testContainerNames(t, "one", []string{"one"})
testContainerNames(t, "one\n", []string{"one"})
testContainerNames(t, "one\ntwo\nthree\n", []string{"one", "two", "three"})
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,885 @@
// This implements a very basic Swift server
// Everything is stored in memory
//
// This comes from the https://github.com/mitchellh/goamz
// and was adapted for Swift
//
package swifttest
import (
"bytes"
"crypto/md5"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"mime"
"net"
"net/http"
"net/url"
"path"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/ncw/swift"
)
const (
DEBUG = false
)
type SwiftServer struct {
t *testing.T
reqId int
mu sync.Mutex
Listener net.Listener
AuthURL string
URL string
Containers map[string]*container
Accounts map[string]*account
Sessions map[string]*session
}
// The Folder type represents a container stored in an account
type Folder struct {
Count int `json:"count"`
Bytes int `json:"bytes"`
Name string `json:"name"`
}
// The Key type represents an item stored in an container.
type Key struct {
Key string `json:"name"`
LastModified string `json:"last_modified"`
Size int64 `json:"bytes"`
// ETag gives the hex-encoded MD5 sum of the contents,
// surrounded with double-quotes.
ETag string `json:"hash"`
ContentType string `json:"content_type"`
// Owner Owner
}
type Subdir struct {
Subdir string `json:"subdir"`
}
type swiftError struct {
statusCode int
Code string
Message string
}
type action struct {
srv *SwiftServer
w http.ResponseWriter
req *http.Request
reqId string
user *account
}
type session struct {
username string
}
type metadata struct {
meta http.Header // metadata to return with requests.
}
type account struct {
swift.Account
metadata
password string
}
type object struct {
metadata
name string
mtime time.Time
checksum []byte // also held as ETag in meta.
data []byte
content_type string
}
type container struct {
metadata
name string
ctime time.Time
objects map[string]*object
bytes int
}
// A resource encapsulates the subject of an HTTP request.
// The resource referred to may or may not exist
// when the request is made.
type resource interface {
put(a *action) interface{}
get(a *action) interface{}
post(a *action) interface{}
delete(a *action) interface{}
copy(a *action) interface{}
}
type objectResource struct {
name string
version string
container *container // always non-nil.
object *object // may be nil.
}
type containerResource struct {
name string
container *container // non-nil if the container already exists.
}
var responseParams = map[string]bool{
"content-type": true,
"content-language": true,
"expires": true,
"cache-control": true,
"content-disposition": true,
"content-encoding": true,
}
func fatalf(code int, codeStr string, errf string, a ...interface{}) {
panic(&swiftError{
statusCode: code,
Code: codeStr,
Message: fmt.Sprintf(errf, a...),
})
}
func (m metadata) setMetadata(a *action, resource string) {
for key, values := range a.req.Header {
key = http.CanonicalHeaderKey(key)
if metaHeaders[key] || strings.HasPrefix(key, "X-"+strings.Title(resource)+"-Meta-") {
if values[0] != "" || resource == "object" {
m.meta[key] = values
} else {
m.meta.Del(key)
}
}
}
}
func (m metadata) getMetadata(a *action) {
h := a.w.Header()
for name, d := range m.meta {
h[name] = d
}
}
func (c container) list(delimiter string, marker string, prefix string, parent string) (resp []interface{}) {
var tmp orderedObjects
// first get all matching objects and arrange them in alphabetical order.
for _, obj := range c.objects {
if strings.HasPrefix(obj.name, prefix) {
tmp = append(tmp, obj)
}
}
sort.Sort(tmp)
var prefixes []string
for _, obj := range tmp {
if !strings.HasPrefix(obj.name, prefix) {
continue
}
isPrefix := false
name := obj.name
if parent != "" {
if path.Dir(obj.name) != path.Clean(parent) {
continue
}
} else if delimiter != "" {
if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 {
name = obj.name[:len(prefix)+i+len(delimiter)]
if prefixes != nil && prefixes[len(prefixes)-1] == name {
continue
}
isPrefix = true
}
}
if name <= marker {
continue
}
if isPrefix {
prefixes = append(prefixes, name)
resp = append(resp, Subdir{
Subdir: name,
})
} else {
resp = append(resp, obj)
}
}
return
}
// GET on a container lists the objects in the container.
func (r containerResource) get(a *action) interface{} {
if r.container == nil {
fatalf(404, "NoSuchContainer", "The specified container does not exist")
}
delimiter := a.req.Form.Get("delimiter")
marker := a.req.Form.Get("marker")
prefix := a.req.Form.Get("prefix")
format := a.req.URL.Query().Get("format")
parent := a.req.Form.Get("path")
a.w.Header().Set("X-Container-Bytes-Used", strconv.Itoa(r.container.bytes))
a.w.Header().Set("X-Container-Object-Count", strconv.Itoa(len(r.container.objects)))
r.container.getMetadata(a)
if a.req.Method == "HEAD" {
return nil
}
objects := r.container.list(delimiter, marker, prefix, parent)
if format == "json" {
a.w.Header().Set("Content-Type", "application/json")
var resp []interface{}
for _, item := range objects {
if obj, ok := item.(*object); ok {
resp = append(resp, obj.Key())
} else {
resp = append(resp, item)
}
}
return resp
} else {
for _, item := range objects {
if obj, ok := item.(*object); ok {
a.w.Write([]byte(obj.name + "\n"))
} else if subdir, ok := item.(Subdir); ok {
a.w.Write([]byte(subdir.Subdir + "\n"))
}
}
return nil
}
}
// orderedContainers holds a slice of containers that can be sorted
// by name.
type orderedContainers []*container
func (s orderedContainers) Len() int {
return len(s)
}
func (s orderedContainers) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
func (s orderedContainers) Less(i, j int) bool {
return s[i].name < s[j].name
}
func (r containerResource) delete(a *action) interface{} {
b := r.container
if b == nil {
fatalf(404, "NoSuchContainer", "The specified container does not exist")
}
if len(b.objects) > 0 {
fatalf(409, "Conflict", "The container you tried to delete is not empty")
}
delete(a.srv.Containers, b.name)
a.user.Containers--
return nil
}
func (r containerResource) put(a *action) interface{} {
if a.req.URL.Query().Get("extract-archive") != "" {
fatalf(403, "Operation forbidden", "Bulk upload is not supported")
}
if r.container == nil {
if !validContainerName(r.name) {
fatalf(400, "InvalidContainerName", "The specified container is not valid")
}
r.container = &container{
name: r.name,
objects: make(map[string]*object),
metadata: metadata{
meta: make(http.Header),
},
}
r.container.setMetadata(a, "container")
a.srv.Containers[r.name] = r.container
a.user.Containers++
}
return nil
}
func (r containerResource) post(a *action) interface{} {
if r.container == nil {
fatalf(400, "Method", "The resource could not be found.")
} else {
r.container.setMetadata(a, "container")
a.w.WriteHeader(201)
jsonMarshal(a.w, Folder{
Count: len(r.container.objects),
Bytes: r.container.bytes,
Name: r.container.name,
})
}
return nil
}
func (containerResource) copy(a *action) interface{} { return notAllowed() }
// validContainerName returns whether name is a valid bucket name.
// Here are the rules, from:
// http://docs.openstack.org/api/openstack-object-storage/1.0/content/ch_object-storage-dev-api-storage.html
//
// Container names cannot exceed 256 bytes and cannot contain the / character.
//
func validContainerName(name string) bool {
if len(name) == 0 || len(name) > 256 {
return false
}
for _, r := range name {
switch {
case r == '/':
return false
default:
}
}
return true
}
// orderedObjects holds a slice of objects that can be sorted
// by name.
type orderedObjects []*object
func (s orderedObjects) Len() int {
return len(s)
}
func (s orderedObjects) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
func (s orderedObjects) Less(i, j int) bool {
return s[i].name < s[j].name
}
func (obj *object) Key() Key {
return Key{
Key: obj.name,
LastModified: obj.mtime.Format("2006-01-02T15:04:05"),
Size: int64(len(obj.data)),
ETag: fmt.Sprintf("%x", obj.checksum),
ContentType: obj.content_type,
}
}
var metaHeaders = map[string]bool{
"Content-Type": true,
"Content-Encoding": true,
"Content-Disposition": true,
"X-Object-Manifest": true,
}
var rangeRegexp = regexp.MustCompile("(bytes=)?([0-9]*)-([0-9]*)")
// GET on an object gets the contents of the object.
func (objr objectResource) get(a *action) interface{} {
var (
etag []byte
reader io.Reader
start int
end int = -1
)
obj := objr.object
if obj == nil {
fatalf(404, "Not Found", "The resource could not be found.")
}
h := a.w.Header()
// add metadata
obj.getMetadata(a)
if r := a.req.Header.Get("Range"); r != "" {
m := rangeRegexp.FindStringSubmatch(r)
if m[2] != "" {
start, _ = strconv.Atoi(m[2])
}
if m[3] != "" {
end, _ = strconv.Atoi(m[3])
}
}
max := func(a int, b int) int {
if a > b {
return a
}
return b
}
if manifest, ok := obj.meta["X-Object-Manifest"]; ok {
var segments []io.Reader
components := strings.SplitN(manifest[0], "/", 2)
segContainer := a.srv.Containers[components[0]]
prefix := components[1]
resp := segContainer.list("", "", prefix, "")
sum := md5.New()
cursor := 0
size := 0
for _, item := range resp {
if obj, ok := item.(*object); ok {
length := len(obj.data)
size += length
sum.Write([]byte(components[0] + "/" + obj.name + "\n"))
if start >= cursor+length {
continue
}
segments = append(segments, bytes.NewReader(obj.data[max(0, start - cursor):]))
cursor += length
}
}
etag = sum.Sum(nil)
if end == -1 {
end = size
}
reader = io.LimitReader(io.MultiReader(segments...), int64(end - start))
} else {
if end == -1 {
end = len(obj.data)
}
etag = obj.checksum
reader = bytes.NewReader(obj.data[start:end])
}
h.Set("Content-Length", fmt.Sprint(end - start))
h.Set("ETag", hex.EncodeToString(etag))
h.Set("Last-Modified", obj.mtime.Format(http.TimeFormat))
if a.req.Method == "HEAD" {
return nil
}
// TODO avoid holding the lock when writing data.
_, err := io.Copy(a.w, reader)
if err != nil {
// we can't do much except just log the fact.
log.Printf("error writing data: %v", err)
}
return nil
}
// PUT on an object creates the object.
func (objr objectResource) put(a *action) interface{} {
var expectHash []byte
if c := a.req.Header.Get("ETag"); c != "" {
var err error
expectHash, err = hex.DecodeString(c)
if err != nil || len(expectHash) != md5.Size {
fatalf(400, "InvalidDigest", "The ETag you specified was invalid")
}
}
sum := md5.New()
// TODO avoid holding lock while reading data.
data, err := ioutil.ReadAll(io.TeeReader(a.req.Body, sum))
if err != nil {
fatalf(400, "TODO", "read error")
}
gotHash := sum.Sum(nil)
if expectHash != nil && bytes.Compare(gotHash, expectHash) != 0 {
fatalf(422, "Bad ETag", "The ETag you specified did not match what we received")
}
if a.req.ContentLength >= 0 && int64(len(data)) != a.req.ContentLength {
fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header")
}
// TODO is this correct, or should we erase all previous metadata?
obj := objr.object
if obj == nil {
obj = &object{
name: objr.name,
metadata: metadata{
meta: make(http.Header),
},
}
a.user.Objects++
} else {
objr.container.bytes -= len(obj.data)
a.user.BytesUsed -= int64(len(obj.data))
}
var content_type string
if content_type = a.req.Header.Get("Content-Type"); content_type == "" {
content_type := mime.TypeByExtension(obj.name)
if content_type == "" {
content_type = "application/octet-stream"
}
}
// PUT request has been successful - save data and metadata
obj.setMetadata(a, "object")
obj.content_type = content_type
obj.data = data
obj.checksum = gotHash
obj.mtime = time.Now().UTC()
objr.container.objects[objr.name] = obj
objr.container.bytes += len(data)
a.user.BytesUsed += int64(len(data))
h := a.w.Header()
h.Set("ETag", hex.EncodeToString(obj.checksum))
return nil
}
func (objr objectResource) delete(a *action) interface{} {
if objr.object == nil {
fatalf(404, "NoSuchKey", "The specified key does not exist.")
}
objr.container.bytes -= len(objr.object.data)
a.user.BytesUsed -= int64(len(objr.object.data))
delete(objr.container.objects, objr.name)
a.user.Objects--
return nil
}
func (objr objectResource) post(a *action) interface{} {
obj := objr.object
obj.setMetadata(a, "object")
return nil
}
func (objr objectResource) copy(a *action) interface{} {
if objr.object == nil {
fatalf(404, "NoSuchKey", "The specified key does not exist.")
}
obj := objr.object
destination := a.req.Header.Get("Destination")
if destination == "" {
fatalf(400, "Bad Request", "You must provide a Destination header")
}
var (
obj2 *object
objr2 objectResource
)
destURL, _ := url.Parse("/v1/AUTH_tk/" + destination)
r := a.srv.resourceForURL(destURL)
switch t := r.(type) {
case objectResource:
objr2 = t
if objr2.object == nil {
obj2 = &object{
name: objr2.name,
metadata: metadata{
meta: make(http.Header),
},
}
a.user.Objects++
} else {
obj2 = objr2.object
objr2.container.bytes -= len(obj2.data)
a.user.BytesUsed -= int64(len(obj2.data))
}
default:
fatalf(400, "Bad Request", "Destination must point to a valid object path")
}
obj2.content_type = obj.content_type
obj2.data = obj.data
obj2.checksum = obj.checksum
obj2.mtime = time.Now()
objr2.container.objects[objr2.name] = obj2
objr2.container.bytes += len(obj.data)
a.user.BytesUsed += int64(len(obj.data))
for key, values := range obj.metadata.meta {
obj2.metadata.meta[key] = values
}
obj2.setMetadata(a, "object")
return nil
}
func (s *SwiftServer) serveHTTP(w http.ResponseWriter, req *http.Request) {
// ignore error from ParseForm as it's usually spurious.
req.ParseForm()
s.mu.Lock()
defer s.mu.Unlock()
if DEBUG {
log.Printf("swifttest %q %q", req.Method, req.URL)
}
a := &action{
srv: s,
w: w,
req: req,
reqId: fmt.Sprintf("%09X", s.reqId),
}
s.reqId++
var r resource
defer func() {
switch err := recover().(type) {
case *swiftError:
w.Header().Set("Content-Type", `text/plain; charset=utf-8`)
http.Error(w, err.Message, err.statusCode)
case nil:
default:
panic(err)
}
}()
var resp interface{}
if req.URL.String() == "/v1.0" {
username := req.Header.Get("x-auth-user")
key := req.Header.Get("x-auth-key")
if acct, ok := s.Accounts[username]; ok {
if acct.password == key {
r := make([]byte, 16)
_, _ = rand.Read(r)
id := fmt.Sprintf("%X", r)
w.Header().Set("X-Storage-Url", s.URL+"/AUTH_"+username)
w.Header().Set("X-Auth-Token", "AUTH_tk"+string(id))
w.Header().Set("X-Storage-Token", "AUTH_tk"+string(id))
s.Sessions[id] = &session{
username: username,
}
return
}
}
panic(notAuthorized())
}
key := req.Header.Get("x-auth-token")
session, ok := s.Sessions[key[7:]]
if !ok {
panic(notAuthorized())
}
a.user = s.Accounts[session.username]
r = s.resourceForURL(req.URL)
switch req.Method {
case "PUT":
resp = r.put(a)
case "GET", "HEAD":
resp = r.get(a)
case "DELETE":
resp = r.delete(a)
case "POST":
resp = r.post(a)
case "COPY":
resp = r.copy(a)
default:
fatalf(400, "MethodNotAllowed", "unknown http request method %q", req.Method)
}
content_type := req.Header.Get("Content-Type")
if resp != nil && req.Method != "HEAD" {
if strings.HasPrefix(content_type, "application/json") ||
req.URL.Query().Get("format") == "json" {
jsonMarshal(w, resp)
} else {
switch r := resp.(type) {
case string:
w.Write([]byte(r))
default:
w.Write(resp.([]byte))
}
}
}
}
func jsonMarshal(w io.Writer, x interface{}) {
if err := json.NewEncoder(w).Encode(x); err != nil {
panic(fmt.Errorf("error marshalling %#v: %v", x, err))
}
}
var pathRegexp = regexp.MustCompile("/v1/AUTH_[a-zA-Z0-9]+(/([^/]+)(/(.*))?)?")
// resourceForURL returns a resource object for the given URL.
func (srv *SwiftServer) resourceForURL(u *url.URL) (r resource) {
m := pathRegexp.FindStringSubmatch(u.Path)
if m == nil {
fatalf(404, "InvalidURI", "Couldn't parse the specified URI")
}
containerName := m[2]
objectName := m[4]
if containerName == "" {
return rootResource{}
}
b := containerResource{
name: containerName,
container: srv.Containers[containerName],
}
if objectName == "" {
return b
}
if b.container == nil {
fatalf(404, "NoSuchContainer", "The specified container does not exist")
}
objr := objectResource{
name: objectName,
version: u.Query().Get("versionId"),
container: b.container,
}
if obj := objr.container.objects[objr.name]; obj != nil {
objr.object = obj
}
return objr
}
// nullResource has error stubs for all resource methods.
type nullResource struct{}
func notAllowed() interface{} {
fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource")
return nil
}
func notAuthorized() interface{} {
fatalf(401, "Unauthorized", "This server could not verify that you are authorized to access the document you requested.")
return nil
}
func (nullResource) put(a *action) interface{} { return notAllowed() }
func (nullResource) get(a *action) interface{} { return notAllowed() }
func (nullResource) post(a *action) interface{} { return notAllowed() }
func (nullResource) delete(a *action) interface{} { return notAllowed() }
func (nullResource) copy(a *action) interface{} { return notAllowed() }
type rootResource struct{}
func (rootResource) put(a *action) interface{} { return notAllowed() }
func (rootResource) get(a *action) interface{} {
marker := a.req.Form.Get("marker")
prefix := a.req.Form.Get("prefix")
format := a.req.URL.Query().Get("format")
h := a.w.Header()
h.Set("X-Account-Bytes-Used", strconv.Itoa(int(a.user.BytesUsed)))
h.Set("X-Account-Container-Count", strconv.Itoa(int(a.user.Containers)))
h.Set("X-Account-Object-Count", strconv.Itoa(int(a.user.Objects)))
// add metadata
a.user.metadata.getMetadata(a)
if a.req.Method == "HEAD" {
return nil
}
var tmp orderedContainers
// first get all matching objects and arrange them in alphabetical order.
for _, container := range a.srv.Containers {
if strings.HasPrefix(container.name, prefix) {
tmp = append(tmp, container)
}
}
sort.Sort(tmp)
resp := make([]Folder, 0)
for _, container := range tmp {
if container.name <= marker {
continue
}
if format == "json" {
resp = append(resp, Folder{
Count: len(container.objects),
Bytes: container.bytes,
Name: container.name,
})
} else {
a.w.Write([]byte(container.name + "\n"))
}
}
if format == "json" {
return resp
} else {
return nil
}
}
func (r rootResource) post(a *action) interface{} {
a.user.metadata.setMetadata(a, "account")
return nil
}
func (rootResource) delete(a *action) interface{} {
if a.req.URL.Query().Get("bulk-delete") == "1" {
fatalf(403, "Operation forbidden", "Bulk delete is not supported")
}
return notAllowed()
}
func (rootResource) copy(a *action) interface{} { return notAllowed() }
func NewSwiftServer(address string) (*SwiftServer, error) {
var (
l net.Listener
err error
)
if strings.Index(address, ":") == -1 {
for port := 1024; port < 65535; port++ {
addr := fmt.Sprintf("%s:%d", address, port)
if l, err = net.Listen("tcp", addr); err == nil {
address = addr
break
}
}
} else {
l, err = net.Listen("tcp", address)
}
if err != nil {
return nil, fmt.Errorf("cannot listen on %s: %v", address, err)
}
server := &SwiftServer{
Listener: l,
AuthURL: "http://" + l.Addr().String() + "/v1.0",
URL: "http://" + l.Addr().String() + "/v1",
Containers: make(map[string]*container),
Accounts: make(map[string]*account),
Sessions: make(map[string]*session),
}
server.Accounts["swifttest"] = &account{
password: "swifttest",
metadata: metadata{
meta: make(http.Header),
},
}
go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
server.serveHTTP(w, req)
}))
return server, nil
}
func (srv SwiftServer) Close() {
srv.Listener.Close()
}

View file

@ -0,0 +1,57 @@
package swift
import (
"io"
"time"
)
// An io.ReadCloser which obeys an idle timeout
type timeoutReader struct {
reader io.ReadCloser
timeout time.Duration
cancel func()
}
// Returns a wrapper around the reader which obeys an idle
// timeout. The cancel function is called if the timeout happens
func newTimeoutReader(reader io.ReadCloser, timeout time.Duration, cancel func()) *timeoutReader {
return &timeoutReader{
reader: reader,
timeout: timeout,
cancel: cancel,
}
}
// Read reads up to len(p) bytes into p
//
// Waits at most for timeout for the read to complete otherwise returns a timeout
func (t *timeoutReader) Read(p []byte) (int, error) {
// FIXME limit the amount of data read in one chunk so as to not exceed the timeout?
// Do the read in the background
type result struct {
n int
err error
}
done := make(chan result, 1)
go func() {
n, err := t.reader.Read(p)
done <- result{n, err}
}()
// Wait for the read or the timeout
select {
case r := <-done:
return r.n, r.err
case <-time.After(t.timeout):
t.cancel()
return 0, TimeoutError
}
panic("unreachable") // for Go 1.0
}
// Close the channel
func (t *timeoutReader) Close() error {
return t.reader.Close()
}
// Check it satisfies the interface
var _ io.ReadCloser = &timeoutReader{}

View file

@ -0,0 +1,107 @@
// This tests TimeoutReader
package swift
import (
"io"
"io/ioutil"
"sync"
"testing"
"time"
)
// An io.ReadCloser for testing
type testReader struct {
sync.Mutex
n int
delay time.Duration
closed bool
}
// Returns n bytes with at time.Duration delay
func newTestReader(n int, delay time.Duration) *testReader {
return &testReader{
n: n,
delay: delay,
}
}
// Returns 1 byte at a time after delay
func (t *testReader) Read(p []byte) (n int, err error) {
if t.n <= 0 {
return 0, io.EOF
}
time.Sleep(t.delay)
p[0] = 'A'
t.Lock()
t.n--
t.Unlock()
return 1, nil
}
// Close the channel
func (t *testReader) Close() error {
t.Lock()
t.closed = true
t.Unlock()
return nil
}
func TestTimeoutReaderNoTimeout(t *testing.T) {
test := newTestReader(3, 10*time.Millisecond)
cancelled := false
cancel := func() {
cancelled = true
}
tr := newTimeoutReader(test, 100*time.Millisecond, cancel)
b, err := ioutil.ReadAll(tr)
if err != nil || string(b) != "AAA" {
t.Fatalf("Bad read %s %s", err, b)
}
if cancelled {
t.Fatal("Cancelled when shouldn't have been")
}
if test.n != 0 {
t.Fatal("Didn't read all")
}
if test.closed {
t.Fatal("Shouldn't be closed")
}
tr.Close()
if !test.closed {
t.Fatal("Should be closed")
}
}
func TestTimeoutReaderTimeout(t *testing.T) {
// Return those bytes slowly so we get an idle timeout
test := newTestReader(3, 100*time.Millisecond)
cancelled := false
cancel := func() {
cancelled = true
}
tr := newTimeoutReader(test, 10*time.Millisecond, cancel)
_, err := ioutil.ReadAll(tr)
if err != TimeoutError {
t.Fatal("Expecting TimeoutError, got", err)
}
if !cancelled {
t.Fatal("Not cancelled when should have been")
}
test.Lock()
n := test.n
test.Unlock()
if n == 0 {
t.Fatal("Read all")
}
if n != 3 {
t.Fatal("Didn't read any")
}
if test.closed {
t.Fatal("Shouldn't be closed")
}
tr.Close()
if !test.closed {
t.Fatal("Should be closed")
}
}

View file

@ -0,0 +1,34 @@
package swift
import (
"io"
"time"
)
// An io.Reader which resets a watchdog timer whenever data is read
type watchdogReader struct {
timeout time.Duration
reader io.Reader
timer *time.Timer
}
// Returns a new reader which will kick the watchdog timer whenever data is read
func newWatchdogReader(reader io.Reader, timeout time.Duration, timer *time.Timer) *watchdogReader {
return &watchdogReader{
timeout: timeout,
reader: reader,
timer: timer,
}
}
// Read reads up to len(p) bytes into p
func (t *watchdogReader) Read(p []byte) (n int, err error) {
// FIXME limit the amount of data read in one chunk so as to not exceed the timeout?
resetTimer(t.timer, t.timeout)
n, err = t.reader.Read(p)
resetTimer(t.timer, t.timeout)
return
}
// Check it satisfies the interface
var _ io.Reader = &watchdogReader{}

View file

@ -0,0 +1,61 @@
// This tests WatchdogReader
package swift
import (
"io/ioutil"
"testing"
"time"
)
// Uses testReader from timeout_reader_test.go
func testWatchdogReaderTimeout(t *testing.T, initialTimeout, watchdogTimeout time.Duration, expectedTimeout bool) {
test := newTestReader(3, 10*time.Millisecond)
timer := time.NewTimer(initialTimeout)
firedChan := make(chan bool)
started := make(chan bool)
go func() {
started <- true
select {
case <-timer.C:
firedChan <- true
}
}()
<-started
wr := newWatchdogReader(test, watchdogTimeout, timer)
b, err := ioutil.ReadAll(wr)
if err != nil || string(b) != "AAA" {
t.Fatalf("Bad read %s %s", err, b)
}
fired := false
select {
case fired = <-firedChan:
default:
}
if expectedTimeout {
if !fired {
t.Fatal("Timer should have fired")
}
} else {
if fired {
t.Fatal("Timer should not have fired")
}
}
}
func TestWatchdogReaderNoTimeout(t *testing.T) {
testWatchdogReaderTimeout(t, 100*time.Millisecond, 100*time.Millisecond, false)
}
func TestWatchdogReaderTimeout(t *testing.T) {
testWatchdogReaderTimeout(t, 5*time.Millisecond, 5*time.Millisecond, true)
}
func TestWatchdogReaderNoTimeoutShortInitial(t *testing.T) {
testWatchdogReaderTimeout(t, 5*time.Millisecond, 100*time.Millisecond, false)
}
func TestWatchdogReaderTimeoutLongInitial(t *testing.T) {
testWatchdogReaderTimeout(t, 100*time.Millisecond, 5*time.Millisecond, true)
}